[AB-xxx] refactoring rest-api to not be a maven project and restructuring python tool file structure

python/components/rest-api/docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.10.13-alpine3.18
+RUN apk --no-cache add msttcorefonts-installer fontconfig && \
+    update-ms-fonts && \
+    fc-cache -f
+ADD wrapper /
+ADD python/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+ADD resources /python/resources
+ADD python /python
+CMD ["/run.sh"]

python/components/rest-api/python/base/image_gen.py

@@ -0,0 +1,18 @@
+from __main__ import app
+
+from PIL import Image, ImageDraw, ImageFont
+from flask import request
+
+from utils import flask_utils
+
+@app.route('/memes/doge/orangeSun/')
+def image_gen():
+    text = request.args.get('text')
+    with Image.open('resources/img/semf_template.jpg') as im:
+        d1 = ImageDraw.Draw(im)
+        text_box_size = (300, 240)
+        W, H = text_box_size
+        font = ImageFont.truetype(f'Impact.ttf', 60)
+        _, _, w, h = d1.textbbox((0, 0), text, font=font)
+        d1.text(((W-w)/2 + 320, (H-h)/2 + 120), text, font=font, fill=(255, 255, 255))
+        return flask_utils.serve_pil_image(im)

python/components/rest-api/python/main.py

@@ -0,0 +1,61 @@
+import logging
+import os
+
+from flask import Flask
+import importlib
+
+FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+logging.basicConfig(encoding='utf-8', level=logging.INFO, format=FORMAT)
+template_dir = os.path.abspath('resources/templates')
+app = Flask(__name__, template_folder=template_dir)
+
+import sys
+sys.path.append("..")
+# loads the api end points
+from base import image_gen
+
+# This code was only done, because "from custom import *" did not work, it seems it did not execute the code in it
+# while the code was valid
+# https://stackoverflow.com/questions/57878744/how-do-i-dynamically-import-all-py-files-from-a-given-directory-and-all-sub-di
+def get_py_files(src):
+    cwd = os.getcwd()
+    py_files = []
+    for root, dirs, files in os.walk(src):
+        for file in files:
+            if file.endswith(".py"):
+                py_files.append(os.path.join(cwd, root, file))
+    return py_files
+
+
+def dynamic_import(module_name, py_path):
+    module_spec = importlib.util.spec_from_file_location(module_name, py_path)
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
+    return module
+
+
+def dynamic_import_from_src(src, star_import=False):
+    my_py_files = get_py_files(src)
+    for py_file in my_py_files:
+        module_name = os.path.split(py_file)[-1].strip(".py")
+        imported_module = dynamic_import(module_name, py_file)
+        if star_import:
+            for obj in dir(imported_module):
+                globals()[obj] = imported_module.__dict__[obj]
+        else:
+            globals()[module_name] = imported_module
+    return
+
+
+if __name__ == "__main__":
+    dynamic_import_from_src("custom", star_import=False)
+
+@app.route('/')
+def hello():
+    return 'Hello, World?'
+
+
+if __name__ == "__main__":
+    from waitress import serve
+
+    serve(app, host="0.0.0.0", port=8080)

python/components/rest-api/python/requirements.txt

@@ -0,0 +1,17 @@
+blinker==1.7.0
+certifi==2023.7.22
+charset-normalizer==3.3.2
+click==8.1.7
+Flask==3.0.0
+idna==3.4
+importlib-metadata==6.8.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+Pillow==10.1.0
+requests==2.31.0
+urllib3==2.0.7
+waitress==2.1.2
+Werkzeug==3.0.1
+zipp==3.17.0
+pytz==2023.3.post1

python/components/rest-api/python/utils/flask_utils.py

@@ -0,0 +1,17 @@
+from io import BytesIO
+
+from flask import send_file
+
+
+def serve_pil_image(pil_img):
+    img_io = BytesIO()
+    pil_img.save(img_io, 'PNG')
+    img_io.seek(0)
+    return send_file(img_io, mimetype='image/png')
+
+
+class ValidationException(Exception):
+    def __init__(self, provided_value, message):
+        self.provided_value = provided_value
+        self.message = message
+        super().__init__(f'{self.message}: provided value: {provided_value}')

python/components/rest-api/wrapper/run.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo "Starting python server..."
+cd python
+python3 -u main.py

python/deployment/installer/config-deployment/Dockerfile

@@ -0,0 +1,41 @@
+FROM ubuntu as base
+MAINTAINER Sheldan
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG liquibase_version=4.3.5
+ARG postgres_driver_version=42.2.14
+# Install prerequisities for Ansible
+RUN apt-get update \
+&& apt-get install -y unzip wget \
+&& rm -rf /var/lib/apt/lists/
+
+# Install liquibase
+RUN mkdir -p /liqiubase \
+&& wget -nv https://github.com/liquibase/liquibase/releases/download/v${liquibase_version}/liquibase-${liquibase_version}.zip -O /tmp/liquibase.zip \
+&& unzip /tmp/liquibase.zip -d /liquibase
+
+RUN mkdir -p /java \
+&& wget -nv https://corretto.aws/downloads/latest/amazon-corretto-8-x64-linux-jdk.tar.gz -O /tmp/java.tar.gz \
+&& tar -xf /tmp/java.tar.gz --strip-components=1 -C /java
+
+# Install postgres driver
+RUN mkdir -p /postgres \
+&& wget -nv https://jdbc.postgresql.org/download/postgresql-${postgres_driver_version}.jar -O /postgres/driver.jar
+
+# Install ansible and required libraries
+
+FROM python:3.7-slim-buster as runtime
+ARG sql_alchemy_version=1.4.46
+ARG jinja_version=3.1.2
+ARG psycopg2_version=2.9.5
+RUN pip3 install --no-cache-dir psycopg2-binary==${psycopg2_version} SQLAlchemy==${sql_alchemy_version} jinja2==${jinja_version}
+COPY --from=base /liquibase /liquibase
+COPY --from=base /postgres /postgres
+COPY --from=base /java /java
+ENV JAVA_HOME=/java/jre
+ADD python /python
+ADD wrapper /
+
+ENV LIQUIBASE_PATH=/liquibase
+ENV POSTGRES_DRIVER=/postgres/driver.jar
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

python/deployment/installer/config-deployment/python/liquibase_deploy.py

@@ -0,0 +1,17 @@
+import subprocess
+import sys
+
+
+def deploy_liquibase(folder, change_log_file, liquibase_path, base_path, db_config):
+    print(f'Deploying liquibase of {change_log_file} in folder {folder}')
+    process_command = [f'{liquibase_path}/liquibase', f'--defaultsFile={change_log_file}', f'--defaultSchemaName={db_config.scheme}', f'--liquibaseSchemaName={db_config.scheme}',  f'--liquibaseCatalogName={db_config.scheme}', '--logLevel=info', 'update']
+    process = subprocess.Popen(process_command,
+                     cwd=f'{base_path}/liquibase-zips/{folder}',
+                     stderr=sys.stderr,
+                     stdout=sys.stdout)
+
+    process.communicate()
+    code = process.returncode
+    if code != 0:
+        print("Liquibased deployment failed.")
+        sys.exit(code)

python/deployment/installer/config-deployment/python/main.py

@@ -0,0 +1,55 @@
+import jinja2
+import json
+import os
+import re
+import liquibase_deploy
+import sys
+from zipfile import ZipFile
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+config_dir = sys.argv[1]
+
+db_config = DbConfig()
+postgres_driver_path = os.getenv('POSTGRES_DRIVER', '/postgres/driver.jar')
+liquibase_path = os.getenv('LIQUIBASE_PATH', '/liquibase')
+script_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+print("Loading versions.")
+with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+    artifact_config = json.load(artifact_config_file)
+
+print("Loading templates")
+templateLoader = jinja2.FileSystemLoader(searchpath="/python/templates")
+templateEnv = jinja2.Environment(loader=templateLoader)
+variable_prefix_pattern = re.compile(r'ABSTRACTO_\w+')
+variables = {}
+for key, val in os.environ.items():
+    if variable_prefix_pattern.match(key):
+        variables[key.lower().replace('_', '')] = val
+
+template = templateEnv.get_template("liquibase.properties.j2")
+
+print("Starting liquibase deployment")
+for liquibase_artifact in artifact_config['liquibase_artifacts']:
+    zip_file = liquibase_artifact['zip']
+    target_folder = config_dir + '/liquibase-zips/' + zip_file
+    with ZipFile(config_dir + 'liquibase-zips/' + zip_file + '.zip', 'r') as liquibase_zip:
+        liquibase_zip.extractall(target_folder)
+    change_log_file = liquibase_artifact['file']
+    liquibase_config_text = template.render(change_log_file=change_log_file, db_host=db_config.host, db_port=db_config.port,
+                                            db_database=db_config.database, db_user=db_config.user, db_password=db_config.password,
+                                            postgres_driver_path=postgres_driver_path, variables=variables)
+    property_path = script_directory + '/templates/liquibase.properties'
+    with open(property_path, 'w') as liquibase_target_properties:
+        liquibase_target_properties.write(liquibase_config_text)
+    liquibase_deploy.deploy_liquibase(zip_file, property_path, liquibase_path, config_dir, db_config)

python/deployment/installer/config-deployment/python/templates/liquibase.properties.j2

@@ -0,0 +1,9 @@
+changeLogFile: {{ change_log_file }}
+driver: org.postgresql.Driver
+url: jdbc:postgresql://{{ db_host }}:{{ db_port }}/{{ db_database }}
+username: {{ db_user }}
+password: {{ db_password }}
+classpath: {{ postgres_driver_path }}
+{% for key, value in variables.items() %}
+parameter.{{ key }}: {{ value }}
+{% endfor %}

python/deployment/installer/config-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code

python/deployment/installer/template-deployment/Dockerfile

@@ -0,0 +1,10 @@
+
+FROM python:3.7-slim-buster as runtime
+MAINTAINER Sheldan
+ARG sql_alchemy_version=1.4.46
+ARG psycopg2_version=2.9.5
+RUN pip3 install --no-cache-dir psycopg2-binary==${psycopg2_version} SQLAlchemy==${sql_alchemy_version}
+ADD python /python
+ADD wrapper /
+
+ENTRYPOINT [ "/bin/sh", "/deploy.sh" ]

python/deployment/installer/template-deployment/python/main.py

@@ -0,0 +1,50 @@
+import json
+import os
+import sys
+import templates_deploy
+from zipfile import ZipFile
+
+
+use_folder = False
+local_folder = None
+config_dir = sys.argv[1]
+if len(sys.argv) == 3:
+    use_folder = True
+    local_folder = sys.argv[2]
+
+
+class DbConfig:
+    def __init__(self):
+        self.host = os.getenv('DB_HOST')
+        self.port = os.getenv('DB_PORT')
+        self.database = os.getenv('DB_NAME')
+        self.user = os.getenv('DB_USER')
+        self.password = os.getenv('DB_PASS')
+        self.scheme = os.getenv('DB_SCHEME')
+
+
+db_config = DbConfig()
+if not use_folder:
+    print("Not deploying with folder.")
+    print("Loading versions.")
+    with open(config_dir + 'artifact_versions.json') as artifact_config_file:
+        artifact_config = json.load(artifact_config_file)
+
+    print("Deploying templates.")
+    for template_artifact in artifact_config['template_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-templates"
+        os.mkdir(folder_name)
+        with ZipFile(config_dir + 'templates/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+    print("Deploying translation templates")
+    for template_artifact in artifact_config['translation_artifacts']:
+        folder_name = config_dir + '/' + template_artifact + "-translations"
+        with ZipFile(config_dir + 'translations/' + template_artifact + '.zip', 'r') as template_zip:
+            template_zip.extractall(folder_name)
+        templates_deploy.deploy_template_folder(db_config, folder_name)
+
+if use_folder:
+    print("Only deploying folder.")
+    templates_deploy.deploy_template_folder(db_config, local_folder)

python/deployment/installer/template-deployment/python/templates_deploy.py

@@ -0,0 +1,31 @@
+import glob
+import os
+import sqlalchemy as db
+from sqlalchemy.sql import text
+
+
+def deploy_template_folder(db_config, folder):
+    engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_config.user, db_config.password, db_config.host, db_config.port, db_config.database))
+
+    if not os.path.isdir(folder):
+        print(f'Given path {folder} was not a folder. Exiting.')
+        exit(1)
+
+    files = glob.glob(folder + '/**/*.ftl', recursive=True)
+    templates = []
+    for file in files:
+        with open(file) as template_file:
+            file_content = template_file.read()
+            template_key = os.path.splitext(os.path.basename(file))[0]
+            template = {'key': template_key, 'content': file_content}
+            print(f'Deployment template {template}')
+            templates.append(template)
+
+    print(f'Deploying {len(templates)} templates from folder {folder}')
+
+    with engine.connect() as con:
+        with con.begin():
+            statement = text(f"""INSERT INTO {db_config.scheme}template(key, content, last_modified) VALUES(:key, :content, NOW()) ON CONFLICT (key) DO UPDATE SET content = :content""")
+
+            for line in templates:
+                con.execute(statement, **line)

python/deployment/installer/template-deployment/wrapper/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Starting deployment."
+
+target_dir=$1
+
+python3 -u python/main.py "${target_dir}"
+exit_code=$?
+
+echo "Finished deployment."
+exit $exit_code