[OPB-37] adding loki log collector

upgrading to newer abstracto version adding some info to readme
2026-01-02 07:44:18 +00:00 · 2021-08-09 23:42:23 +02:00
parent 0380720ec8
commit 3f959f3bbb
15 changed files with 173 additions and 15 deletions
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -34,5 +34,5 @@ jobs:
        env:
          REGISTRY_PREFIX: docker.pkg.github.com/sheldan/oneplusbot/
          VERSION: ${{ env.version }}
-          ABSTRACTO_VERSION: 1.3.6
+          ABSTRACTO_VERSION: 1.3.7
          ABSTRACTO_REGISTRY_PREFIX: docker.pkg.github.com/sheldan/abstracto/
--- a/README.md
+++ b/README.md
@@ -1,11 +1,20 @@
 # r/oneplus Discord server bot

-This repository contains the rewrite of the [Bot](https://github.com/Rithari/OnePlusBot), which is still in use. The features will be gradually be ported 
-and most of them will be in [abstracto](https://github.com/Sheldan/abstracto), but some of the features need to be customized, as they are r/oneplus specific.
+This repository contains the rewrite of the [Bot](https://github.com/Rithari/OnePlusBot). It is developed by customizing [abstracto](https://github.com/Sheldan/abstracto), because some features are r/oneplus specific.

 The migration of the existing data from the database is handled via one time migration, and can be found [here](https://github.com/Sheldan/OnePlusBot-migration).

+The FAQ configuration can be found [here](https://github.com/Sheldan/OnePlusBot-faq/).
+
 Custom features which were ported
 - [x] FAQ
 - [x] Setup channel handling
 - [x] Referral link handling 
+
+# Technologies used in addition to the ones provided in abstracto
+- [grafana](https://github.com/grafana/grafana) for visualization of the bot status and metrics
+- [Loki](https://github.com/grafana/loki) to visualize and query log files
+- [pgAdmin](https://github.com/postgres/pgadmin4) to view the database
+- [prometheus](https://github.com/prometheus/prometheus) for metric collection
+- [postgres](https://github.com/postgres/postgres) as a database
+  
--- a/deployment/docker-compose/src/main/resources/.env
+++ b/deployment/docker-compose/src/main/resources/.env
@@ -20,6 +20,7 @@ REST_PASSWORD=password
 # port grafana will be reachable
 GRAFANA_PORT=3000
 # port prometheus will be reachable
+LOKI_PORT=3100
 PROMETHEUS_PORT=9090
 # port pg admin will be reachable
 PGADMIN_PORT=5050
@@ -28,4 +29,4 @@ PGADMIN_DEFAULT_EMAIL=sheldan@sheldan.dev
 PGADMIN_DEFAULT_PASSWORD=admin
 TOKEN=<INSERT TOKEN>
 YOUTUBE_API_KEY=<INSERT KEY>
-ONEPLUS_BOT_VERSION=1.5.5
+ONEPLUS_BOT_VERSION=1.5.6
--- a/deployment/docker-compose/src/main/resources/docker-compose.yml
+++ b/deployment/docker-compose/src/main/resources/docker-compose.yml
@@ -109,6 +109,28 @@ services:
      - grafana-user-data:/var/lib/grafana
    networks:
      - oneplusbot
+  promtail:
+    container_name: promtail
+    image: ${REGISTRY_PREFIX}oneplus_bot_promtail:${ONEPLUS_BOT_VERSION}
+    depends_on:
+      - bot
+    restart: unless-stopped
+    command: -config.file=/mnt/config/promtail-config.yaml
+    volumes:
+      - bot-logs:/logs
+    networks:
+      - oneplusbot
+  loki:
+    container_name: loki
+    image: ${REGISTRY_PREFIX}oneplus_bot_loki:${ONEPLUS_BOT_VERSION}
+    depends_on:
+      - promtail
+    command: -config.file=/mnt/config/loki-config.yaml
+    ports:
+      - "127.0.0.1:${LOKI_PORT}:3100"
+    restart: unless-stopped
+    networks:
+      - oneplusbot

 networks:
  oneplusbot:
--- a/deployment/image-packaging/src/main/docker/.env
+++ b/deployment/image-packaging/src/main/docker/.env
@@ -0,0 +1,6 @@
+PROMTAIL_VERSION=2.2.1
+LOKI_VERSION=2.2.1
+PROMETHEUS_VERSION=v2.28.1
+PG_ADMIN_VERSION=5.5
+GRAFANA_VERSION=8.0.6
+POSTGRES_VERSION=13.3-buster
--- a/deployment/image-packaging/src/main/docker/database/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/database/Dockerfile
@@ -1,4 +1,5 @@
-FROM postgres
+ARG POSTGRES_VERSION
+FROM postgres:${POSTGRES_VERSION}
 MAINTAINER Sheldan
 VOLUME /tmp
 ADD sql/init.sql /docker-entrypoint-initdb.d/init.sql
--- a/deployment/image-packaging/src/main/docker/docker-compose.yml
+++ b/deployment/image-packaging/src/main/docker/docker-compose.yml
@@ -2,13 +2,20 @@ version: "3.7"

 services:
  bot:
-    build: oneplusbot
+    build:
+      context: oneplusbot
    image: ${REGISTRY_PREFIX}oneplus_bot:${VERSION:-latest}
  database:
-    build: database
+    build:
+      context: database
+      args:
+        POSTGRES_VERSION: ${POSTGRES_VERSION}
    image: ${REGISTRY_PREFIX}oneplus_bot_database:${VERSION:-latest}
  pg_admin:
-    build: pgAdmin
+    build:
+      context: pgAdmin
+      args:
+        PG_ADMIN_VERSION: ${PG_ADMIN_VERSION}
    image: ${REGISTRY_PREFIX}oneplus_bot_pg_admin:${VERSION:-latest}
  deployment_container:
    build:
@@ -18,8 +25,26 @@ services:
        ABSTRACTO_VERSION: ${ABSTRACTO_VERSION}
    image: ${REGISTRY_PREFIX}oneplus_bot_deployment:${VERSION:-latest}
  prometheus:
-    build: prometheus
+    build:
+      context: prometheus
+      args:
+        PROMETHEUS_VERSION: ${PROMETHEUS_VERSION}
    image: ${REGISTRY_PREFIX}oneplus_bot_prometheus:${VERSION:-latest}
  grafana:
-    build: grafana
+    build:
+      context: grafana
+      args:
+        GRAFANA_VERSION: ${GRAFANA_VERSION}
    image: ${REGISTRY_PREFIX}oneplus_bot_grafana:${VERSION:-latest}
+  promtail:
+    build:
+      context: promtail
+      args:
+        PROMTAIL_VERSION: ${PROMTAIL_VERSION}
+    image: ${REGISTRY_PREFIX}oneplus_bot_promtail:${VERSION:-latest}
+  loki:
+    build:
+      context: loki
+      args:
+        LOKI_VERSION: ${LOKI_VERSION}
+    image: ${REGISTRY_PREFIX}oneplus_bot_loki:${VERSION:-latest}
--- a/deployment/image-packaging/src/main/docker/grafana/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/grafana/Dockerfile
@@ -1,4 +1,5 @@
-FROM grafana/grafana
+ARG GRAFANA_VERSION
+FROM grafana/grafana:${GRAFANA_VERSION}
 MAINTAINER Sheldan
 ADD ./provisioning /etc/grafana/provisioning
 ADD ./config.ini /etc/grafana/config.ini
--- a/deployment/image-packaging/src/main/docker/loki/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/loki/Dockerfile
@@ -0,0 +1,4 @@
+ARG LOKI_VERSION
+FROM grafana/loki:${LOKI_VERSION}
+MAINTAINER Sheldan
+ADD loki.yml /mnt/config/loki-config.yaml
--- a/deployment/image-packaging/src/main/docker/loki/loki.yml
+++ b/deployment/image-packaging/src/main/docker/loki/loki.yml
@@ -0,0 +1,66 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h
+  max_chunk_age: 1h
+  chunk_target_size: 1048576
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 360h
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /tmp/loki/rules
+  rule_path: /tmp/loki/rules-temp
+  ring:
+    kvstore:
+      store: inmemory
--- a/deployment/image-packaging/src/main/docker/pgAdmin/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/pgAdmin/Dockerfile
@@ -1,4 +1,5 @@
-FROM dpage/pgadmin4
+ARG PG_ADMIN_VERSION
+FROM dpage/pgadmin4:${PG_ADMIN_VERSION}
 MAINTAINER Sheldan
 VOLUME /tmp
 ADD config/servers.json /pgadmin4/servers.json
--- a/deployment/image-packaging/src/main/docker/prometheus/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/prometheus/Dockerfile
@@ -1,3 +1,4 @@
-FROM prom/prometheus
+ARG PROMETHEUS_VERSION
+FROM prom/prometheus:${PROMETHEUS_VERSION}
 MAINTAINER Sheldan
 ADD prometheus.yml /etc/prometheus/prometheus.yml
--- a/deployment/image-packaging/src/main/docker/promtail/Dockerfile
+++ b/deployment/image-packaging/src/main/docker/promtail/Dockerfile
@@ -0,0 +1,4 @@
+ARG PROMTAIL_VERSION
+FROM grafana/promtail:${PROMTAIL_VERSION}
+MAINTAINER Sheldan
+ADD promtail.yaml /mnt/config/promtail-config.yaml
--- a/deployment/image-packaging/src/main/docker/promtail/promtail.yaml
+++ b/deployment/image-packaging/src/main/docker/promtail/promtail.yaml
@@ -0,0 +1,17 @@
+server:
+  disable: true
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: oneplus-bot-logs
+          __path__: /logs/*.log
--- a/pom.xml
+++ b/pom.xml
@@ -19,8 +19,8 @@
        <maven.compiler.source>1.8</maven.compiler.source>
        <!-- edit in release.yml as well -->
        <!-- when releasing a new opbot version, update the .env as well-->
-        <abstracto.version>1.3.6</abstracto.version>
-        <abstracto.templates.version>1.2.18</abstracto.templates.version>
+        <abstracto.version>1.3.7</abstracto.version>
+        <abstracto.templates.version>1.2.19</abstracto.templates.version>
    </properties>

    <modules>