[SIS-xxx] making quote attachment URL column larger

changing migration script for starboard/quotes

.env

@@ -1,4 +1,4 @@
 REGISTRY_PREFIX=harbor.sheldan.dev/sissi/
 ABSTRACTO_PREFIX=harbor.sheldan.dev/abstracto/
-VERSION=1.4.55
+VERSION=1.4.56
 ABSTRACTO_VERSION=1.5.38

application/executable/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application</groupId>
         <artifactId>application</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>executable</artifactId>

application/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi</groupId>
         <artifactId>sissi</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-customizations/image-generation-custom/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application.module.custom</groupId>
         <artifactId>sissi-customizations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
 
     <artifactId>image-generation-custom</artifactId>

application/sissi-customizations/moderation-custom/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application.module.custom</groupId>
         <artifactId>sissi-customizations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-customizations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>application</artifactId>
         <groupId>dev.sheldan.sissi.application</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-modules/debra/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application</groupId>
         <artifactId>sissi-modules</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-modules/meetup/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application</groupId>
         <artifactId>sissi-modules</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-modules/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application</groupId>
         <artifactId>application</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-modules/quotes/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.application</groupId>
         <artifactId>sissi-modules</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/collection.xml

@@ -0,0 +1,6 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd"  >
+    <include file="tables/tables.xml" relativeToChangelogFile="true"/>
+</databaseChangeLog>

application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/tables/quote_attachment.xml

@@ -0,0 +1,11 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd"  >
+    <changeSet author="Sheldan" id="quote_attachment-longer_attachment_url">
+        <modifyDataType columnName="url"
+                         newDataType="VARCHAR(4096)"
+                         tableName="quote_attachment"/>
+    </changeSet>
+
+</databaseChangeLog>

application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/tables/tables.xml

@@ -0,0 +1,6 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd"  >
+    <include file="quote_attachment.xml" relativeToChangelogFile="true"/>
+</databaseChangeLog>

application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml

@@ -4,4 +4,5 @@
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd"  >
     <include file="1.0.2/collection.xml" relativeToChangelogFile="true"/>
     <include file="1.4.56/collection.xml" relativeToChangelogFile="true"/>
+    <include file="1.4.57/collection.xml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

application/sissi-modules/rss-news/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>sissi-modules</artifactId>
         <groupId>dev.sheldan.sissi.application</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

deployment/helm/sissi/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v2
 name: sissi
 description: A Helm chart for Kubernetes
 type: application
-version: 1.4.55
+version: 1.4.56

deployment/helm/sissi/values.yaml

@@ -3,7 +3,7 @@ bot:
   repository: harbor.sheldan.dev/sissi
   pullPolicy: Always
   image: sissi-bot
-  tag: 1.4.55
+  tag: 1.4.56
   livenessProbe:
     initialDelaySeconds: 60
     periodSeconds: 5
@@ -20,7 +20,7 @@ restApi:
   repository: harbor.sheldan.dev/sissi
   pullPolicy: Always
   image: sissi-rest-api
-  tag: 1.4.55
+  tag: 1.4.56
   podAnnotations: {}
   podSecurityContext: {}
   securityContext: {}
@@ -60,7 +60,7 @@ privateRestApi:
   repository: harbor.sheldan.dev/sissi
   pullPolicy: Always
   image: sissi-private-rest-api
-  tag: 1.4.55
+  tag: 1.4.56
   podAnnotations: {}
   podSecurityContext: {}
   securityContext: {}
@@ -96,7 +96,7 @@ templateDeploymentData:
   repository: harbor.sheldan.dev/sissi
   pullPolicy: Always
   image: sissi-template-data
-  tag: 1.4.55
+  tag: 1.4.56
 dbConfigDeployment:
   enabled: true
   repository: harbor.sheldan.dev/abstracto
@@ -107,7 +107,7 @@ dbConfigDeploymentData:
   repository: harbor.sheldan.dev/sissi
   pullPolicy: Always
   image: sissi-db-data
-  tag: 1.4.55
+  tag: 1.4.56
 dbCredentials:
   host: null
   port: null

deployment/image-packaging/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi</groupId>
         <artifactId>deployment</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

deployment/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi</groupId>
         <artifactId>sissi</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

pom.xml

@@ -13,7 +13,7 @@
     <groupId>dev.sheldan.sissi</groupId>
     <artifactId>sissi</artifactId>
     <name>Sissi</name>
-    <version>1.4.56</version>
+    <version>1.4.57-SNAPSHOT</version>
 
     <properties>
         <maven.compiler.target>17</maven.compiler.target>
@@ -72,7 +72,7 @@
         <connection>scm:git:${project.scm.url}</connection>
         <developerConnection>scm:git:${project.scm.url}</developerConnection>
         <url>https://github.com/Sheldan/Sissi.git</url>
-        <tag>sissi-1.4.56</tag>
+        <tag>HEAD</tag>
     </scm>
 
 </project>

python/tools/migrations/quotes-starboard-import/main.py

@@ -14,13 +14,17 @@ engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password,
 
 with engine.connect() as con:
     posts = load_all_starboard_posts(con)
-    print(posts)
+    sub_posts = chunks = [posts[x:x+100] for x in range(0, len(posts), 100)]
-    print(f'Loaded {len(posts)}')
+    print(f'Loaded {len(posts)} into {len(sub_posts)} partitions')
-    enriched_posts = enrich_posts(posts)
+    counter = 0
-    print(f'Enriched posts')
+    for sub_post in sub_posts:
-    import_quotes(enriched_posts, con)
+        print(f'Partition size {len(sub_post)}')
-    print(f'Done storing quotes')
+        enriched_posts = enrich_posts(sub_post)
-    con.commit()
+        print(f'Enriched posts')
-    fix_quote_created(enriched_posts, con)
+        import_quotes(enriched_posts, con)
-    con.commit()
+        print(f'Done storing quotes')
-    print('Done.')
+        con.commit()
+        fix_quote_created(enriched_posts, con)
+        con.commit()
+        counter += 1
+        print(f'Done. {counter}')

python/tools/migrations/quotes-starboard-import/post_loader.py

@@ -12,7 +12,7 @@ def enrich_posts(posts):
         print(f"Loading post {post['message_id']}")
         url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}"
         message = requests.get(url, headers={'Authorization': token})
-        time.sleep(5)
+        time.sleep(0.5)
         if message.status_code == 200:
             message_obj = json.loads(message.content)
             post['content'] = message_obj['content']

python/tools/migrations/quotes-starboard-import/quote_importer.py

@@ -4,6 +4,7 @@ def import_quotes(posts, con):
     for post in posts:
         if 'content' not in post:
             print(f"Skipping {post['message_id']} because no content, did it fail?")
+            continue
         print(f"Inserting {post['message_id']}")
         statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id, 
                                                         server_id, message_id, text, created) 

python/tools/migrations/quotes-starboard-import/starboard_loader.py

@@ -2,17 +2,18 @@ from sqlalchemy.sql import text
 
 
 def load_all_starboard_posts(conn):
-    squery = text("""select sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
+    squery = text("""select distinct sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
 from starboard_post sp
-inner join starboard_post_reaction spr
+         inner join starboard_post_reaction spr
-on sp.id = spr.post_id
+                    on sp.id = spr.post_id
-and spr.created = (
+                        and spr.reactor_user_in_server_id = (
-select spr.created 
+                            select reactor_user_in_server_id
-from starboard_post_reaction spr2 
+                            from starboard_post_reaction spr2
-where spr2.post_id = sp.id 
+                            where spr2.post_id = sp.id
-order by created limit 1
+                            order by created limit 1
-                 )
+                        )
- where sp.ignored = false
+where sp.ignored = false
+and sp.post_message_id not in (select message_id from quote)
  """)
     rs = conn.execute(squery)
     found_posts = []

templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi</groupId>
         <artifactId>sissi</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/customization-templates/image-generation-customization-templates/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>customization-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
 
     <artifactId>image-generation-customization-templates</artifactId>

templates/sissi-templates/customization-templates/moderation-customization-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>customization-templates</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/customization-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>sissi-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/module-templates/debra-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>module-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/module-templates/meetup-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>module-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/module-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>sissi-templates</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/module-templates/quotes-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>module-templates</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/module-templates/rss-news-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>module-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>templates</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/template-overrides/experience-template-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>template-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/template-overrides/logging-template-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>template-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/template-overrides/moderation-template-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>template-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-templates/template-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>sissi-templates</artifactId>
         <groupId>dev.sheldan.sissi.templates</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/customization-translations/moderation-customization-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
         <artifactId>customization-translations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/customization-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>sissi-translations</artifactId>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/module-translations/debra-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>module-translations</artifactId>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/module-translations/meetup-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
         <artifactId>module-translations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/module-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
         <artifactId>sissi-translations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/module-translations/quote-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
         <artifactId>module-translations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/module-translations/quote-translations/src/main/resources/en_US/config/feature_starboardQuoteSync_en_US.ftl

@@ -0,0 +1 @@
+Quote-Starboard sync

templates/sissi-translations/module-translations/rss-news-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>module-translations</artifactId>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates</groupId>
         <artifactId>templates</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/translation-overrides/experience-translation-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
         <artifactId>translation-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/translation-overrides/logging-translation-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
         <artifactId>translation-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/translation-overrides/moderation-translation-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
         <artifactId>translation-overrides</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

templates/sissi-translations/translation-overrides/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>dev.sheldan.sissi.templates.translations</groupId>
         <artifactId>sissi-translations</artifactId>
-        <version>1.4.56</version>
+        <version>1.4.57-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>