[AB-xxx] restructuring installer module to be outside of maven

split liquibase and template deployment
adding various features to the deployment images

.github/workflows/release.yml

@@ -37,15 +37,19 @@ jobs:
           branch: master
           ssh-key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
           folder: abstracto-application/documentation/target/generated-docs
-      - name: Login to GitHub Packages Docker Registry
-        uses: docker/login-action@v2
+      - name: Login to Harbor
+          uses: docker/login-action@v2
+          with:
+            registry: harbor.sheldan.dev
+            username: ${{ secrets.HARBOR_USERNAME }}
+            password: ${{ secrets.HARBOR_TOKEN }}
+      - name: Load env files
+        id: dotenv
+        uses: falti/dotenv-action@v1.0.4
         with:
-          registry: docker.pkg.github.com
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Push deployment container
-        working-directory: ./abstracto-application/installer/src/main/docker/deployment
+          path: deployment/.env
+      - name: Push container
         run: docker-compose build && docker-compose push
         env:
-          REGISTRY_PREFIX: docker.pkg.github.com/sheldan/abstracto/
-          VERSION: ${{ env.version }}
+          REGISTRY_PREFIX: ${{ steps.dotenv.outputs.registry_prefix }}
+          VERSION: ${{ steps.dotenv.outputs.version }}

abstracto-application/installer/pom.xml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <groupId>dev.sheldan.abstracto</groupId>
-        <artifactId>abstracto-application</artifactId>
-        <version>1.5.2-SNAPSHOT</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>installer</artifactId>
-    <packaging>pom</packaging>
-
-
-</project>

abstracto-application/installer/src/main/docker/deployment/python/liquibase_deploy.py

@@ -1,16 +0,0 @@
-import subprocess
-import sys
-
-
-def deploy_liquibase(folder, change_log_file, liquibase_path):
-    print("Deploying liquibase of %s in folder %s" % (change_log_file, folder))
-    process = subprocess.Popen(['%s/liquibase' %  (liquibase_path), '--defaultsFile=%s' % (change_log_file), '--liquibaseSchemaName=abstracto',  '--liquibaseCatalogName=abstracto', '--logLevel=info', 'update'],
-                     cwd='liquibase-zips/%s' % (folder),
-                     stderr=sys.stderr,
-                     stdout=sys.stdout)
-
-    process.communicate()
-    code = process.returncode
-    if code != 0:
-        print("Liquibased deployment failed.")
-        sys.exit(code)

abstracto-application/installer/src/main/docker/deployment/python/main.py

@@ -1,85 +0,0 @@
-import jinja2
-import json
-import liquibase_deploy
-import os
-import sys
-import re
-import templates_deploy
-from zipfile import ZipFile
-
-if len(sys.argv) < 3:
-    sys.exit('Wrong amount of parameters.')
-
-use_folder = False
-local_folder = None
-if len(sys.argv) == 4:
-    use_folder = True
-    local_folder = sys.argv[3]
-
-deploy_templates = sys.argv[1] == 'yes'
-deploy_liquibase = sys.argv[2] == 'yes'
-
-class DbConfig:
-    def __init__(self):
-        self.host = os.getenv('DB_HOST')
-        self.port = os.getenv('DB_PORT')
-        self.database = os.getenv('DB_NAME')
-        self.user = os.getenv('DB_USER')
-        self.password = os.getenv('DB_PASS')
-
-db_config = DbConfig()
-if not use_folder:
-    print("Not deploying with folder.")
-    postgres_driver_path = os.getenv('POSTGRES_DRIVER')
-    liquibase_path = os.getenv('LIQUIBASE_PATH')
-    print("Loading versions.")
-    with open('artifact_versions.json') as artifact_config_file:
-        artifact_config = json.load(artifact_config_file)
-
-
-    print("Loading templates")
-    templateLoader = jinja2.FileSystemLoader(searchpath="/python/templates")
-    templateEnv = jinja2.Environment(loader=templateLoader)
-    variable_prefix_pattern = re.compile(r'ABSTRACTO_\w+')
-    variables = {}
-    for key, val in os.environ.items():
-        if variable_prefix_pattern.match(key):
-            variables[key.lower().replace('_', '')] = val
-
-    template = templateEnv.get_template("liquibase.properties.j2")
-
-    if deploy_liquibase:
-        print("Starting liquibase deployment")
-        for liquibase_artifact in artifact_config['liquibase_artifacts']:
-            zip_file = liquibase_artifact['zip']
-            target_folder = '/liquibase-zips/' + zip_file
-            with ZipFile('liquibase-zips/' + zip_file + '.zip', 'r') as liquibase_zip:
-                liquibase_zip.extractall(target_folder)
-            change_log_file = liquibase_artifact['file']
-            liquibase_config_text = template.render(change_log_file=change_log_file, db_host=db_config.host, db_port=db_config.port,
-                                                    db_database=db_config.database, db_user=db_config.user, db_password=db_config.password,
-                                                    postgres_driver_path=postgres_driver_path, variables=variables)
-            property_path = target_folder + '/liquibase.properties'
-            with open(property_path, 'w') as liquibase_target_properties:
-                liquibase_target_properties.write(liquibase_config_text)
-            liquibase_deploy.deploy_liquibase(zip_file, property_path, liquibase_path)
-
-    if deploy_templates:
-        print("Deploying templates.")
-        for template_artifact in artifact_config['template_artifacts']:
-            folder_name = template_artifact + "-templates"
-            os.mkdir(folder_name)
-            with ZipFile('templates/' + template_artifact + '.zip', 'r') as template_zip:
-                template_zip.extractall(folder_name)
-            templates_deploy.deploy_template_folder(db_config, folder_name)
-
-        print("Deploying translation templates")
-        for template_artifact in artifact_config['translation_artifacts']:
-            folder_name = template_artifact + "-translations"
-            with ZipFile('translations/' + template_artifact + '.zip', 'r') as template_zip:
-                template_zip.extractall(folder_name)
-            templates_deploy.deploy_template_folder(db_config, folder_name)
-
-if use_folder:
-    print("Only deploying folder.")
-    templates_deploy.deploy_template_folder(db_config, local_folder)

abstracto-application/installer/src/main/docker/deployment/wrapper/deploy.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-
-DEPLOY_LIQUIBASE=no
-DEPLOY_TEMPLATES=no
-
-echo "Starting deployment."
-
-if [ "x$EXECUTE_LIQUIBASE" = 'xtrue' ]; then
-  DEPLOY_LIQUIBASE=yes
-fi
-
-if [ "x$EXECUTE_TEMPLATES" = 'xtrue' ]; then
-  DEPLOY_TEMPLATES=yes
-fi
-exit_code=0
-if [ "x$EXECUTE_DEPLOYMENT" = 'xtrue' ]; then
-   python3 -u python/main.py $DEPLOY_TEMPLATES $DEPLOY_LIQUIBASE
-   exit_code=$?
-fi
-
-echo "Finished deployment."
-exit $exit_code

abstracto-application/installer/src/main/docker/docker-compose.yaml

@@ -1,6 +0,0 @@
-version: "3.7"
-
-services:
-  bot:
-    build: deployment
-    image: ${REGISTRY_PREFIX}abstracto_deployment:${VERSION:-latest}

abstracto-application/pom.xml

@@ -18,7 +18,6 @@
         <module>abstracto-modules</module>
         <module>scheduling</module>
         <module>documentation</module>
-        <module>installer</module>
         <module>bundle</module>
     </modules>
 

abstracto-application/scheduling/scheduling-impl/src/main/resources/scheduling.properties

@@ -9,7 +9,7 @@ spring.quartz.properties.org.quartz.threadPool.threadCount=4
 spring.quartz.properties.org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
 spring.quartz.properties.org.quartz.jobStore.useProperties=true
 spring.quartz.properties.org.quartz.jobStore.misfireThreshold=45000
-spring.quartz.properties.org.quartz.jobStore.tablePrefix=qrtz_
+spring.quartz.properties.org.quartz.jobStore.tablePrefix=abstracto.qrtz_
 spring.quartz.properties.org.quartz.jobStore.isClustered=false
 spring.quartz.properties.org.quartz.scheduler.classLoadHelper.class = org.quartz.simpl.CascadingClassLoadHelper
 spring.quartz.properties.org.quartz.plugin.shutdownHook.class=org.quartz.plugins.management.ShutdownHookPlugin

deployment/installer/.env

@@ -0,0 +1,2 @@
+REGISTRY_PREFIX=harbor.sheldan.dev/abstracto/
+VERSION=0.0.1

deployment/installer/config-deployment/Dockerfile

@@ -33,12 +33,9 @@ COPY --from=base /liquibase /liquibase
 COPY --from=base /postgres /postgres
 COPY --from=base /java /java
 ENV JAVA_HOME=/java/jre
-ADD python/ /python
-ADD wrapper/ /
-RUN chmod +x /deploy.sh
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.7.3/wait /wait
-RUN chmod +x /wait
+ADD python /python
+ADD wrapper /
 
 ENV LIQUIBASE_PATH=/liquibase
 ENV POSTGRES_DRIVER=/postgres/driver.jar
-CMD /wait && /deploy.sh
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

deployment/installer/config-deployment/python/liquibase_deploy.py

@@ -0,0 +1,17 @@
+import subprocess
+import sys
+
+
+def deploy_liquibase(folder, change_log_file, liquibase_path, base_path, db_config):
+    print(f'Deploying liquibase of {change_log_file} in folder {folder}')
+    process_command = [f'{liquibase_path}/liquibase', f'--defaultsFile={change_log_file}', f'--defaultSchemaName={db_config.scheme}', f'--liquibaseSchemaName={db_config.scheme}',  f'--liquibaseCatalogName={db_config.scheme}', '--logLevel=info', 'update']
+    process = subprocess.Popen(process_command,
+                     cwd=f'{base_path}/liquibase-zips/{folder}',
+                     stderr=sys.stderr,
+                     stdout=sys.stdout)
+
+    process.communicate()
+    code = process.returncode
+    if code != 0:
+        print("Liquibased deployment failed.")
+        sys.exit(code)

deployment/installer/config-deployment/python/main.py

@@ -0,0 +1,55 @@
+import jinja2
+import json
+import os
+import re
+import liquibase_deploy
+import sys
+from zipfile import ZipFile
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+config_dir = sys.argv[1]
+
+db_config = DbConfig()
+postgres_driver_path = os.getenv('POSTGRES_DRIVER', '/postgres/driver.jar')
+liquibase_path = os.getenv('LIQUIBASE_PATH', '/liquibase')
+script_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+print("Loading versions.")
+with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+    artifact_config = json.load(artifact_config_file)
+
+print("Loading templates")
+templateLoader = jinja2.FileSystemLoader(searchpath="/python/templates")
+templateEnv = jinja2.Environment(loader=templateLoader)
+variable_prefix_pattern = re.compile(r'ABSTRACTO_\w+')
+variables = {}
+for key, val in os.environ.items():
+    if variable_prefix_pattern.match(key):
+        variables[key.lower().replace('_', '')] = val
+
+template = templateEnv.get_template("liquibase.properties.j2")
+
+print("Starting liquibase deployment")
+for liquibase_artifact in artifact_config['liquibase_artifacts']:
+    zip_file = liquibase_artifact['zip']
+    target_folder = config_dir + '/liquibase-zips/' + zip_file
+    with ZipFile(config_dir + 'liquibase-zips/' + zip_file + '.zip', 'r') as liquibase_zip:
+        liquibase_zip.extractall(target_folder)
+    change_log_file = liquibase_artifact['file']
+    liquibase_config_text = template.render(change_log_file=change_log_file, db_host=db_config.host, db_port=db_config.port,
+                                            db_database=db_config.database, db_user=db_config.user, db_password=db_config.password,
+                                            postgres_driver_path=postgres_driver_path, variables=variables)
+    property_path = script_directory + '/templates/liquibase.properties'
+    with open(property_path, 'w') as liquibase_target_properties:
+        liquibase_target_properties.write(liquibase_config_text)
+    liquibase_deploy.deploy_liquibase(zip_file, property_path, liquibase_path, config_dir, db_config)

deployment/installer/config-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code

deployment/installer/docker-compose.yaml

@@ -0,0 +1,9 @@
+version: "3.7"
+
+services:
+  config_deployment:
+    build: config-deployment
+    image: ${REGISTRY_PREFIX}abstracto-db-deployment:${VERSION:-latest}
+  template_deployment:
+    build: template-deployment
+    image: ${REGISTRY_PREFIX}abstracto-template-deployment:${VERSION:-latest}

deployment/installer/template-deployment/Dockerfile

@@ -0,0 +1,10 @@
+
+FROM python:3.7-slim-buster as runtime
+MAINTAINER Sheldan
+ARG sql_alchemy_version=1.4.46
+ARG psycopg2_version=2.9.5
+RUN pip3 install --no-cache-dir psycopg2-binary==${psycopg2_version} SQLAlchemy==${sql_alchemy_version}
+ADD python /python
+ADD wrapper /
+
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

deployment/installer/template-deployment/python/main.py

@@ -0,0 +1,50 @@
+import json
+import os
+import sys
+import templates_deploy
+from zipfile import ZipFile
+
+
+use_folder = False
+local_folder = None
+config_dir = sys.argv[1]
+if len(sys.argv) == 3:
+    use_folder = True
+    local_folder = sys.argv[2]
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+db_config = DbConfig()
+if not use_folder:
+    print("Not deploying with folder.")
+    print("Loading versions.")
+    with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+        artifact_config = json.load(artifact_config_file)
+
+    print("Deploying templates.")
+    for template_artifact in artifact_config['template_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-templates"
+        os.mkdir(folder_name)
+        with ZipFile(config_dir + 'templates/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+    print("Deploying translation templates")
+    for template_artifact in artifact_config['translation_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-translations"
+        with ZipFile(config_dir + 'translations/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+if use_folder:
+    print("Only deploying folder.")
+    templates_deploy.deploy_template_folder(db_config, local_folder)

deployment/installer/template-deployment/python/templates_deploy.py

@@ -8,7 +8,7 @@ def deploy_template_folder(db_config, folder):
     engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_config.user, db_config.password, db_config.host, db_config.port, db_config.database))
 
     if not os.path.isdir(folder):
-        print("Given path was not a folder. Exiting.")
+        print(f'Given path {folder} was not a folder. Exiting.')
         exit(1)
 
     files = glob.glob(folder + '/**/*.ftl', recursive=True)
@@ -18,14 +18,14 @@ def deploy_template_folder(db_config, folder):
             file_content = template_file.read()
             template_key = os.path.splitext(os.path.basename(file))[0]
             template = {'key': template_key, 'content': file_content}
-            print('Deployment template %s', template)
+            print(f'Deployment template {template}')
             templates.append(template)
 
-    print('Deploying %s templates from folder %s' % (len(templates), folder))
+    print(f'Deploying {len(templates)} templates from folder {folder}')
 
     with engine.connect() as con:
         with con.begin():
-            statement = text("""INSERT INTO template(key, content, last_modified) VALUES(:key, :content, NOW()) ON CONFLICT (key) DO UPDATE SET content = :content""")
+            statement = text(f"""INSERT INTO {db_config.scheme}template(key, content, last_modified) VALUES(:key, :content, NOW()) ON CONFLICT (key) DO UPDATE SET content = :content""")
 
             for line in templates:
                 con.execute(statement, **line)

deployment/installer/template-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code