[OPB-37] adding loki log collector

upgrading to newer abstracto version
adding some info to readme

.github/workflows/release.yml

@@ -34,5 +34,5 @@ jobs:
         env:
           REGISTRY_PREFIX: docker.pkg.github.com/sheldan/oneplusbot/
           VERSION: ${{ env.version }}
-          ABSTRACTO_VERSION: 1.3.6
+          ABSTRACTO_VERSION: 1.3.7
           ABSTRACTO_REGISTRY_PREFIX: docker.pkg.github.com/sheldan/abstracto/

README.md

@@ -1,11 +1,20 @@
 # r/oneplus Discord server bot
 
-This repository contains the rewrite of the [Bot](https://github.com/Rithari/OnePlusBot), which is still in use. The features will be gradually be ported 
+This repository contains the rewrite of the [Bot](https://github.com/Rithari/OnePlusBot). It is developed by customizing [abstracto](https://github.com/Sheldan/abstracto), because some features are r/oneplus specific.
-and most of them will be in [abstracto](https://github.com/Sheldan/abstracto), but some of the features need to be customized, as they are r/oneplus specific.
 
 The migration of the existing data from the database is handled via one time migration, and can be found [here](https://github.com/Sheldan/OnePlusBot-migration).
 
+The FAQ configuration can be found [here](https://github.com/Sheldan/OnePlusBot-faq/).
+
 Custom features which were ported
  - [x] FAQ
  - [x] Setup channel handling
  - [x] Referral link handling 
+
+# Technologies used in addition to the ones provided in abstracto
+- [grafana](https://github.com/grafana/grafana) for visualization of the bot status and metrics
+- [Loki](https://github.com/grafana/loki) to visualize and query log files
+- [pgAdmin](https://github.com/postgres/pgadmin4) to view the database
+- [prometheus](https://github.com/prometheus/prometheus) for metric collection
+- [postgres](https://github.com/postgres/postgres) as a database
+  

deployment/docker-compose/src/main/resources/.env

@@ -20,6 +20,7 @@ REST_PASSWORD=password
 # port grafana will be reachable
 GRAFANA_PORT=3000
 # port prometheus will be reachable
+LOKI_PORT=3100
 PROMETHEUS_PORT=9090
 # port pg admin will be reachable
 PGADMIN_PORT=5050
@@ -28,4 +29,4 @@ PGADMIN_DEFAULT_EMAIL=sheldan@sheldan.dev
 PGADMIN_DEFAULT_PASSWORD=admin
 TOKEN=<INSERT TOKEN>
 YOUTUBE_API_KEY=<INSERT KEY>
-ONEPLUS_BOT_VERSION=1.5.5
+ONEPLUS_BOT_VERSION=1.5.6

deployment/docker-compose/src/main/resources/docker-compose.yml

@@ -109,6 +109,28 @@ services:
       - grafana-user-data:/var/lib/grafana
     networks:
       - oneplusbot
+  promtail:
+    container_name: promtail
+    image: ${REGISTRY_PREFIX}oneplus_bot_promtail:${ONEPLUS_BOT_VERSION}
+    depends_on:
+      - bot
+    restart: unless-stopped
+    command: -config.file=/mnt/config/promtail-config.yaml
+    volumes:
+      - bot-logs:/logs
+    networks:
+      - oneplusbot
+  loki:
+    container_name: loki
+    image: ${REGISTRY_PREFIX}oneplus_bot_loki:${ONEPLUS_BOT_VERSION}
+    depends_on:
+      - promtail
+    command: -config.file=/mnt/config/loki-config.yaml
+    ports:
+      - "127.0.0.1:${LOKI_PORT}:3100"
+    restart: unless-stopped
+    networks:
+      - oneplusbot
 
 networks:
   oneplusbot:

deployment/image-packaging/src/main/docker/.env

@@ -0,0 +1,6 @@
+PROMTAIL_VERSION=2.2.1
+LOKI_VERSION=2.2.1
+PROMETHEUS_VERSION=v2.28.1
+PG_ADMIN_VERSION=5.5
+GRAFANA_VERSION=8.0.6
+POSTGRES_VERSION=13.3-buster

deployment/image-packaging/src/main/docker/database/Dockerfile

@@ -1,4 +1,5 @@
-FROM postgres
+ARG POSTGRES_VERSION
+FROM postgres:${POSTGRES_VERSION}
 MAINTAINER Sheldan
 VOLUME /tmp
 ADD sql/init.sql /docker-entrypoint-initdb.d/init.sql

deployment/image-packaging/src/main/docker/docker-compose.yml

@@ -2,13 +2,20 @@ version: "3.7"
 
 services:
   bot:
-    build: oneplusbot
+    build:
+      context: oneplusbot
     image: ${REGISTRY_PREFIX}oneplus_bot:${VERSION:-latest}
   database:
-    build: database
+    build:
+      context: database
+      args:
+        POSTGRES_VERSION: ${POSTGRES_VERSION}
     image: ${REGISTRY_PREFIX}oneplus_bot_database:${VERSION:-latest}
   pg_admin:
-    build: pgAdmin
+    build:
+      context: pgAdmin
+      args:
+        PG_ADMIN_VERSION: ${PG_ADMIN_VERSION}
     image: ${REGISTRY_PREFIX}oneplus_bot_pg_admin:${VERSION:-latest}
   deployment_container:
     build:
@@ -18,8 +25,26 @@ services:
         ABSTRACTO_VERSION: ${ABSTRACTO_VERSION}
     image: ${REGISTRY_PREFIX}oneplus_bot_deployment:${VERSION:-latest}
   prometheus:
-    build: prometheus
+    build:
+      context: prometheus
+      args:
+        PROMETHEUS_VERSION: ${PROMETHEUS_VERSION}
     image: ${REGISTRY_PREFIX}oneplus_bot_prometheus:${VERSION:-latest}
   grafana:
-    build: grafana
+    build:
+      context: grafana
+      args:
+        GRAFANA_VERSION: ${GRAFANA_VERSION}
     image: ${REGISTRY_PREFIX}oneplus_bot_grafana:${VERSION:-latest}
+  promtail:
+    build:
+      context: promtail
+      args:
+        PROMTAIL_VERSION: ${PROMTAIL_VERSION}
+    image: ${REGISTRY_PREFIX}oneplus_bot_promtail:${VERSION:-latest}
+  loki:
+    build:
+      context: loki
+      args:
+        LOKI_VERSION: ${LOKI_VERSION}
+    image: ${REGISTRY_PREFIX}oneplus_bot_loki:${VERSION:-latest}

deployment/image-packaging/src/main/docker/grafana/Dockerfile

@@ -1,4 +1,5 @@
-FROM grafana/grafana
+ARG GRAFANA_VERSION
+FROM grafana/grafana:${GRAFANA_VERSION}
 MAINTAINER Sheldan
 ADD ./provisioning /etc/grafana/provisioning
 ADD ./config.ini /etc/grafana/config.ini

deployment/image-packaging/src/main/docker/loki/Dockerfile

@@ -0,0 +1,4 @@
+ARG LOKI_VERSION
+FROM grafana/loki:${LOKI_VERSION}
+MAINTAINER Sheldan
+ADD loki.yml /mnt/config/loki-config.yaml

deployment/image-packaging/src/main/docker/loki/loki.yml

@@ -0,0 +1,66 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h
+  max_chunk_age: 1h
+  chunk_target_size: 1048576
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 360h
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /tmp/loki/rules
+  rule_path: /tmp/loki/rules-temp
+  ring:
+    kvstore:
+      store: inmemory

deployment/image-packaging/src/main/docker/pgAdmin/Dockerfile

@@ -1,4 +1,5 @@
-FROM dpage/pgadmin4
+ARG PG_ADMIN_VERSION
+FROM dpage/pgadmin4:${PG_ADMIN_VERSION}
 MAINTAINER Sheldan
 VOLUME /tmp
 ADD config/servers.json /pgadmin4/servers.json

deployment/image-packaging/src/main/docker/prometheus/Dockerfile

@@ -1,3 +1,4 @@
-FROM prom/prometheus
+ARG PROMETHEUS_VERSION
+FROM prom/prometheus:${PROMETHEUS_VERSION}
 MAINTAINER Sheldan
 ADD prometheus.yml /etc/prometheus/prometheus.yml

deployment/image-packaging/src/main/docker/promtail/Dockerfile

@@ -0,0 +1,4 @@
+ARG PROMTAIL_VERSION
+FROM grafana/promtail:${PROMTAIL_VERSION}
+MAINTAINER Sheldan
+ADD promtail.yaml /mnt/config/promtail-config.yaml

deployment/image-packaging/src/main/docker/promtail/promtail.yaml

@@ -0,0 +1,17 @@
+server:
+  disable: true
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: oneplus-bot-logs
+          __path__: /logs/*.log

pom.xml

@@ -19,8 +19,8 @@
         <maven.compiler.source>1.8</maven.compiler.source>
         <!-- edit in release.yml as well -->
         <!-- when releasing a new opbot version, update the .env as well-->
-        <abstracto.version>1.3.6</abstracto.version>
+        <abstracto.version>1.3.7</abstracto.version>
-        <abstracto.templates.version>1.2.18</abstracto.templates.version>
+        <abstracto.templates.version>1.2.19</abstracto.templates.version>
     </properties>
 
     <modules>