[AB-xxx] refactoring rest-api to not be a maven project and restructuring python tool file structure

python/deployment/installer/config-deployment/Dockerfile

@@ -0,0 +1,41 @@
+FROM ubuntu as base
+MAINTAINER Sheldan
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG liquibase_version=4.3.5
+ARG postgres_driver_version=42.2.14
+# Install prerequisities for Ansible
+RUN apt-get update \
+&& apt-get install -y unzip wget \
+&& rm -rf /var/lib/apt/lists/
+
+# Install liquibase
+RUN mkdir -p /liqiubase \
+&& wget -nv https://github.com/liquibase/liquibase/releases/download/v${liquibase_version}/liquibase-${liquibase_version}.zip -O /tmp/liquibase.zip \
+&& unzip /tmp/liquibase.zip -d /liquibase
+
+RUN mkdir -p /java \
+&& wget -nv https://corretto.aws/downloads/latest/amazon-corretto-8-x64-linux-jdk.tar.gz -O /tmp/java.tar.gz \
+&& tar -xf /tmp/java.tar.gz --strip-components=1 -C /java
+
+# Install postgres driver
+RUN mkdir -p /postgres \
+&& wget -nv https://jdbc.postgresql.org/download/postgresql-${postgres_driver_version}.jar -O /postgres/driver.jar
+
+# Install ansible and required libraries
+
+FROM python:3.7-slim-buster as runtime
+ARG sql_alchemy_version=1.4.46
+ARG jinja_version=3.1.2
+ARG psycopg2_version=2.9.5
+RUN pip3 install --no-cache-dir psycopg2-binary==${psycopg2_version} SQLAlchemy==${sql_alchemy_version} jinja2==${jinja_version}
+COPY --from=base /liquibase /liquibase
+COPY --from=base /postgres /postgres
+COPY --from=base /java /java
+ENV JAVA_HOME=/java/jre
+ADD python /python
+ADD wrapper /
+
+ENV LIQUIBASE_PATH=/liquibase
+ENV POSTGRES_DRIVER=/postgres/driver.jar
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

python/deployment/installer/config-deployment/python/liquibase_deploy.py

@@ -0,0 +1,17 @@
+import subprocess
+import sys
+
+
+def deploy_liquibase(folder, change_log_file, liquibase_path, base_path, db_config):
+    print(f'Deploying liquibase of {change_log_file} in folder {folder}')
+    process_command = [f'{liquibase_path}/liquibase', f'--defaultsFile={change_log_file}', f'--defaultSchemaName={db_config.scheme}', f'--liquibaseSchemaName={db_config.scheme}',  f'--liquibaseCatalogName={db_config.scheme}', '--logLevel=info', 'update']
+    process = subprocess.Popen(process_command,
+                     cwd=f'{base_path}/liquibase-zips/{folder}',
+                     stderr=sys.stderr,
+                     stdout=sys.stdout)
+
+    process.communicate()
+    code = process.returncode
+    if code != 0:
+        print("Liquibased deployment failed.")
+        sys.exit(code)

python/deployment/installer/config-deployment/python/main.py

@@ -0,0 +1,55 @@
+import jinja2
+import json
+import os
+import re
+import liquibase_deploy
+import sys
+from zipfile import ZipFile
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+config_dir = sys.argv[1]
+
+db_config = DbConfig()
+postgres_driver_path = os.getenv('POSTGRES_DRIVER', '/postgres/driver.jar')
+liquibase_path = os.getenv('LIQUIBASE_PATH', '/liquibase')
+script_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+print("Loading versions.")
+with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+    artifact_config = json.load(artifact_config_file)
+
+print("Loading templates")
+templateLoader = jinja2.FileSystemLoader(searchpath="/python/templates")
+templateEnv = jinja2.Environment(loader=templateLoader)
+variable_prefix_pattern = re.compile(r'ABSTRACTO_\w+')
+variables = {}
+for key, val in os.environ.items():
+    if variable_prefix_pattern.match(key):
+        variables[key.lower().replace('_', '')] = val
+
+template = templateEnv.get_template("liquibase.properties.j2")
+
+print("Starting liquibase deployment")
+for liquibase_artifact in artifact_config['liquibase_artifacts']:
+    zip_file = liquibase_artifact['zip']
+    target_folder = config_dir + '/liquibase-zips/' + zip_file
+    with ZipFile(config_dir + 'liquibase-zips/' + zip_file + '.zip', 'r') as liquibase_zip:
+        liquibase_zip.extractall(target_folder)
+    change_log_file = liquibase_artifact['file']
+    liquibase_config_text = template.render(change_log_file=change_log_file, db_host=db_config.host, db_port=db_config.port,
+                                            db_database=db_config.database, db_user=db_config.user, db_password=db_config.password,
+                                            postgres_driver_path=postgres_driver_path, variables=variables)
+    property_path = script_directory + '/templates/liquibase.properties'
+    with open(property_path, 'w') as liquibase_target_properties:
+        liquibase_target_properties.write(liquibase_config_text)
+    liquibase_deploy.deploy_liquibase(zip_file, property_path, liquibase_path, config_dir, db_config)

python/deployment/installer/config-deployment/python/templates/liquibase.properties.j2

@@ -0,0 +1,9 @@
+changeLogFile: {{ change_log_file }}
+driver: org.postgresql.Driver
+url: jdbc:postgresql://{{ db_host }}:{{ db_port }}/{{ db_database }}
+username: {{ db_user }}
+password: {{ db_password }}
+classpath: {{ postgres_driver_path }}
+{% for key, value in variables.items() %}
+parameter.{{ key }}: {{ value }}
+{% endfor %}

python/deployment/installer/config-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code

python/deployment/installer/template-deployment/Dockerfile

@@ -0,0 +1,10 @@
+
+FROM python:3.7-slim-buster as runtime
+MAINTAINER Sheldan
+ARG sql_alchemy_version=1.4.46
+ARG psycopg2_version=2.9.5
+RUN pip3 install --no-cache-dir psycopg2-binary==${psycopg2_version} SQLAlchemy==${sql_alchemy_version}
+ADD python /python
+ADD wrapper /
+
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

python/deployment/installer/template-deployment/python/main.py

@@ -0,0 +1,50 @@
+import json
+import os
+import sys
+import templates_deploy
+from zipfile import ZipFile
+
+
+use_folder = False
+local_folder = None
+config_dir = sys.argv[1]
+if len(sys.argv) == 3:
+    use_folder = True
+    local_folder = sys.argv[2]
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+db_config = DbConfig()
+if not use_folder:
+    print("Not deploying with folder.")
+    print("Loading versions.")
+    with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+        artifact_config = json.load(artifact_config_file)
+
+    print("Deploying templates.")
+    for template_artifact in artifact_config['template_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-templates"
+        os.mkdir(folder_name)
+        with ZipFile(config_dir + 'templates/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+    print("Deploying translation templates")
+    for template_artifact in artifact_config['translation_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-translations"
+        with ZipFile(config_dir + 'translations/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+if use_folder:
+    print("Only deploying folder.")
+    templates_deploy.deploy_template_folder(db_config, local_folder)

python/deployment/installer/template-deployment/python/templates_deploy.py

@@ -0,0 +1,31 @@
+import glob
+import os
+import sqlalchemy as db
+from sqlalchemy.sql import text
+
+
+def deploy_template_folder(db_config, folder):
+    engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_config.user, db_config.password, db_config.host, db_config.port, db_config.database))
+
+    if not os.path.isdir(folder):
+        print(f'Given path {folder} was not a folder. Exiting.')
+        exit(1)
+
+    files = glob.glob(folder + '/**/*.ftl', recursive=True)
+    templates = []
+    for file in files:
+        with open(file) as template_file:
+            file_content = template_file.read()
+            template_key = os.path.splitext(os.path.basename(file))[0]
+            template = {'key': template_key, 'content': file_content}
+            print(f'Deployment template {template}')
+            templates.append(template)
+
+    print(f'Deploying {len(templates)} templates from folder {folder}')
+
+    with engine.connect() as con:
+        with con.begin():
+            statement = text(f"""INSERT INTO {db_config.scheme}template(key, content, last_modified) VALUES(:key, :content, NOW()) ON CONFLICT (key) DO UPDATE SET content = :content""")
+
+            for line in templates:
+                con.execute(statement, **line)

python/deployment/installer/template-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code