[RAB-2] adding code to retrieve legacy quotes

adding migration script for legacy quotes
changing docker-compose file to use different container names

tools/quotes-migration/.gitignore

@@ -0,0 +1 @@
+*.db

tools/quotes-migration/dto.py

@@ -0,0 +1,17 @@
+
+class LegacyQuoteAttachment:
+    message_id = 0
+    file_name = ''
+    url = ''
+    is_image: bool = False
+
+
+class LegacyQuote:
+    id = 0
+    channel_id = 0
+    author_id = 0
+    adder_id = 0
+    creation_time_stamp = None
+    content = ''
+    message_id = 0
+    attachments = None

tools/quotes-migration/legacy_loader.py

@@ -0,0 +1,54 @@
+import sqlite3
+from sqlite3 import Error
+import datetime
+
+from dto import LegacyQuote, LegacyQuoteAttachment
+
+
+def create_connection(file: str):
+    conn = None
+    try:
+        conn = sqlite3.connect(file)
+    except Error as e:
+        print(e)
+    return conn
+
+
+def load_all_quotes_with_attachments(conn):
+    cur = conn.cursor()
+    cur.execute("SELECT q.id, q.chan_id, q.author_id, "
+                "q.adder_id, q.jump_url, q.'timestamp', q.content,"
+                "a.msg_id, a.filename, a.url, a.is_image  "
+                "FROM quotes q left outer join attachments a ON q.jump_url like '%' || a.msg_id")
+
+    rows = cur.fetchall()
+
+    quotes = {}
+    for row in rows:
+        quote_id = row[0]
+        if quote_id not in quotes:
+            quotes[quote_id] = LegacyQuote()
+        current_quote = quotes[quote_id]
+        current_quote.id = quote_id
+        current_quote.channel_id = row[1]
+        current_quote.author_id = row[2]
+        current_quote.adder_id = row[3]
+        if len(row[5]) > 25:
+            current_quote.creation_time_stamp = datetime.datetime.strptime(row[5], '%Y-%m-%d %H:%M:%S.%f')
+        elif len(row[5]) != 19:
+            current_quote.creation_time_stamp = datetime.datetime.strptime(row[5], '%Y-%m-%d %H:%M:%S.%f')
+        else:
+            current_quote.creation_time_stamp = datetime.datetime.strptime(row[5], '%Y-%m-%d %H:%M:%S')
+        current_quote.content = row[6]
+        current_quote.message_id = row[4][row[4].rindex('/')+1:]
+
+        if row[7] is not None:
+            if current_quote.attachments is None:
+                current_quote.attachments = []
+            attachment = LegacyQuoteAttachment()
+            attachment.message_id = row[7]
+            attachment.file_name = row[8]
+            attachment.url = row[9]
+            attachment.is_image = row[10] == 1
+            current_quote.attachments.append(attachment)
+    return quotes

tools/quotes-migration/main.py

@@ -0,0 +1,22 @@
+from legacy_loader import create_connection, load_all_quotes_with_attachments
+from quote_importer import insert_quotes, create_users, create_channels
+import sqlalchemy as db
+import os
+
+server_id = os.getenv('SERVER_ID')
+conn = create_connection('new_quotes.db')
+
+all_quotes = load_all_quotes_with_attachments(conn)
+db_host = os.getenv('DB_HOST')
+db_port = os.getenv('DB_PORT')
+db_database = os.getenv('DB_NAME')
+db_user = os.getenv('DB_USER')
+db_password = os.getenv('DB_PASS')
+engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password, db_host, db_port, db_database))
+
+with engine.connect() as con:
+    with con.begin():
+        create_users(server_id, all_quotes, con)
+        create_channels(server_id, all_quotes, con)
+    with con.begin():
+        insert_quotes(server_id, all_quotes, con)

tools/quotes-migration/quote_importer.py

@@ -0,0 +1,70 @@
+from sqlalchemy.sql import text
+
+
+def insert_quotes(server_id: int, quotes, con):
+    for quote_id in quotes:
+        quote = quotes[quote_id]
+        statement = text("""INSERT INTO quote(id, server_id, author_user_in_server_id, adder_user_in_server_id, source_channel_id, message_id, text, created) 
+                            VALUES(:id, :server_id, 
+                            (select user_in_server_id from user_in_server where user_id = :author_id and server_id = :server_id), 
+                            (select user_in_server_id from user_in_server where user_id = :adder_id and server_id = :server_id), 
+                            :channel_id, :message_id, :text, :created)""")
+        con.execute(statement, {'id': quote.id, 'server_id': server_id, 'author_id': quote.author_id, 'adder_id': quote.adder_id,
+                                'channel_id': quote.channel_id, 'message_id': quote.message_id,
+                                'text': quote.content, 'created': quote.creation_time_stamp})
+        if quote.attachments:
+            attachment_statement = text("""INSERT INTO quote_attachment(quote_id, server_id, url, is_image) 
+                                    VALUES(:quote_id, :server_id, :url, :is_image)""")
+            for attachment in quote.attachments:
+                con.execute(attachment_statement, {'quote_id': quote_id, 'server_id': server_id, 'url': attachment.url, 'is_image': attachment.is_image})
+
+
+def create_channels(server_id: int, quotes, con):
+    channel_ids = {}
+    for quote_id in quotes:
+        quote = quotes[quote_id]
+        if not does_channel_exist(quote.channel_id, con) and quote.channel_id not in channel_ids:
+            channel_ids[quote.channel_id] = 1
+    for channel_id in channel_ids:
+        create_channel(channel_id, server_id, con)
+
+
+def create_users(server_id: int, quotes, con):
+    created_users = {}
+    for quote_id in quotes:
+        quote = quotes[quote_id]
+        if not does_user_exist(quote.adder_id, con) and quote.adder_id not in created_users:
+            create_user(quote.adder_id, con)
+            create_user_in_server(quote.adder_id, server_id, con)
+            created_users[quote.adder_id] = 1
+        if not does_user_exist(quote.author_id, con) and quote.author_id not in created_users:
+            create_user(quote.author_id, con)
+            create_user_in_server(quote.author_id, server_id, con)
+            created_users[quote.author_id] = 1
+
+
+def does_user_exist(user_id, con):
+    statement = text("""SELECT count(1) FROM auser where id = :id""")
+    return con.execute(statement, {'id': user_id}).fetchone()[0] == 1
+
+
+def does_channel_exist(channel_id, con):
+    statement = text("""SELECT count(1) FROM channel where id = :id""")
+    return con.execute(statement, {'id': channel_id}).fetchone()[0] == 1
+
+
+def create_user(user_id, con):
+    statement = text("""INSERT INTO auser(id) VALUES(:id)""")
+    print(f'Creating user {user_id}')
+    con.execute(statement, {'id': user_id})
+
+
+def create_channel(channel_id, server_id, con):
+    statement = text("""INSERT INTO channel(id, server_id, type, deleted) VALUES(:id, :server_id, 'TEXT', false)""")
+    print(f'Creating channel {channel_id}')
+    con.execute(statement, {'id': channel_id, 'server_id': server_id})
+
+
+def create_user_in_server(user_id, server_id, con):
+    statement = text("""INSERT INTO user_in_server(server_id, user_id) VALUES(:server_id, :user_id) returning user_in_server_id""")
+    return con.execute(statement, {'user_id': user_id, 'server_id': server_id}).fetchone()[0]