[SIS-xxx] making quote attachment URL column larger

changing migration script for starboard/quotes

python/tools/migrations/quotes-starboard-import/main.py

@@ -14,13 +14,17 @@ engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password,
 
 with engine.connect() as con:
     posts = load_all_starboard_posts(con)
-    print(posts)
-    print(f'Loaded {len(posts)}')
-    enriched_posts = enrich_posts(posts)
-    print(f'Enriched posts')
-    import_quotes(enriched_posts, con)
-    print(f'Done storing quotes')
-    con.commit()
-    fix_quote_created(enriched_posts, con)
-    con.commit()
-    print('Done.')
+    sub_posts = chunks = [posts[x:x+100] for x in range(0, len(posts), 100)]
+    print(f'Loaded {len(posts)} into {len(sub_posts)} partitions')
+    counter = 0
+    for sub_post in sub_posts:
+        print(f'Partition size {len(sub_post)}')
+        enriched_posts = enrich_posts(sub_post)
+        print(f'Enriched posts')
+        import_quotes(enriched_posts, con)
+        print(f'Done storing quotes')
+        con.commit()
+        fix_quote_created(enriched_posts, con)
+        con.commit()
+        counter += 1
+        print(f'Done. {counter}')

python/tools/migrations/quotes-starboard-import/post_loader.py

@@ -12,7 +12,7 @@ def enrich_posts(posts):
         print(f"Loading post {post['message_id']}")
         url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}"
         message = requests.get(url, headers={'Authorization': token})
-        time.sleep(5)
+        time.sleep(0.5)
         if message.status_code == 200:
             message_obj = json.loads(message.content)
             post['content'] = message_obj['content']

python/tools/migrations/quotes-starboard-import/quote_importer.py

@@ -4,6 +4,7 @@ def import_quotes(posts, con):
     for post in posts:
         if 'content' not in post:
             print(f"Skipping {post['message_id']} because no content, did it fail?")
+            continue
         print(f"Inserting {post['message_id']}")
         statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id, 
                                                         server_id, message_id, text, created) 

python/tools/migrations/quotes-starboard-import/starboard_loader.py

@@ -2,17 +2,18 @@ from sqlalchemy.sql import text
 
 
 def load_all_starboard_posts(conn):
-    squery = text("""select sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
+    squery = text("""select distinct sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
 from starboard_post sp
-inner join starboard_post_reaction spr
-on sp.id = spr.post_id
-and spr.created = (
-select spr.created 
-from starboard_post_reaction spr2 
-where spr2.post_id = sp.id 
-order by created limit 1
-                 )
- where sp.ignored = false
+         inner join starboard_post_reaction spr
+                    on sp.id = spr.post_id
+                        and spr.reactor_user_in_server_id = (
+                            select reactor_user_in_server_id
+                            from starboard_post_reaction spr2
+                            where spr2.post_id = sp.id
+                            order by created limit 1
+                        )
+where sp.ignored = false
+and sp.post_message_id not in (select message_id from quote)
  """)
     rs = conn.execute(squery)
     found_posts = []