Compare commits

...

13 Commits

Author SHA1 Message Date
release-bot
df49c313a4 [maven-release-plugin] prepare release sissi-1.4.58 2024-08-06 21:06:09 +00:00
Sheldan
68bf6b5062 [SIS-xxx] upgrading to abstracto 1.5.44 2024-08-06 23:03:07 +02:00
Sheldan
29eb743c35 [SIS-xxx] upgrading to abstracto 1.5.41 2024-08-02 22:47:52 +02:00
Sheldan
03f9b64db2 [SIS-xxx] changing github actions image version to a concrete version and upgrading to docker compose 2024-08-02 22:37:04 +02:00
Sheldan
212bc09d90 [SIS-xxx] upgrading to abstracto snapshot version
adding template for sending message level action
2024-07-27 19:53:27 +02:00
release-bot
e1f327a687 Commit from GitHub Actions (Publishes a new version of Sissi) 2024-07-12 22:58:34 +00:00
release-bot
7af0d2db1c [maven-release-plugin] prepare for next development iteration 2024-07-12 22:54:33 +00:00
release-bot
b4f740fcf6 [maven-release-plugin] prepare release sissi-1.4.57 2024-07-12 22:54:31 +00:00
Sheldan
dda7ed7db8 [SIS-xxx] making quote attachment URL column larger
changing migration script for starboard/quotes
2024-07-13 00:51:41 +02:00
Sheldan
af006ee880 [SIS-xxx] adding missed starboard sync feature template 2024-07-12 23:05:32 +02:00
release-bot
125febc02a Commit from GitHub Actions (Publishes a new version of Sissi) 2024-07-12 20:55:49 +00:00
release-bot
13f92247cf [maven-release-plugin] prepare for next development iteration 2024-07-12 20:50:49 +00:00
release-bot
79ed55d988 [maven-release-plugin] prepare release sissi-1.4.56 2024-07-12 20:50:47 +00:00
54 changed files with 108 additions and 76 deletions

4
.env
View File

@@ -1,4 +1,4 @@
REGISTRY_PREFIX=harbor.sheldan.dev/sissi/
ABSTRACTO_PREFIX=harbor.sheldan.dev/abstracto/
VERSION=1.4.55
ABSTRACTO_VERSION=1.5.38
VERSION=1.4.57
ABSTRACTO_VERSION=1.5.44

View File

@@ -14,7 +14,7 @@ on:
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v3

View File

@@ -5,7 +5,7 @@ permissions:
contents: write
jobs:
publish:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
with:
@@ -72,7 +72,7 @@ jobs:
with:
path: .env
- name: Build and push Docker containers
run: docker-compose build && docker-compose push
run: docker compose build && docker compose push
env:
REGISTRY_PREFIX: ${{ steps.dotenv.outputs.registry_prefix }}
VERSION: ${{ steps.dotenv.outputs.version }}

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application</groupId>
<artifactId>application</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>executable</artifactId>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi</groupId>
<artifactId>sissi</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -4,7 +4,7 @@
<parent>
<groupId>dev.sheldan.sissi.application.module.custom</groupId>
<artifactId>sissi-customizations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<artifactId>image-generation-custom</artifactId>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application.module.custom</groupId>
<artifactId>sissi-customizations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>application</artifactId>
<groupId>dev.sheldan.sissi.application</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application</groupId>
<artifactId>sissi-modules</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application</groupId>
<artifactId>sissi-modules</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application</groupId>
<artifactId>application</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.application</groupId>
<artifactId>sissi-modules</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -0,0 +1,6 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="tables/tables.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>

View File

@@ -0,0 +1,11 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<changeSet author="Sheldan" id="quote_attachment-longer_attachment_url">
<modifyDataType columnName="url"
newDataType="VARCHAR(4096)"
tableName="quote_attachment"/>
</changeSet>
</databaseChangeLog>

View File

@@ -0,0 +1,6 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="quote_attachment.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>

View File

@@ -4,4 +4,5 @@
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="1.0.2/collection.xml" relativeToChangelogFile="true"/>
<include file="1.4.56/collection.xml" relativeToChangelogFile="true"/>
<include file="1.4.57/collection.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>sissi-modules</artifactId>
<groupId>dev.sheldan.sissi.application</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -2,4 +2,4 @@ apiVersion: v2
name: sissi
description: A Helm chart for Kubernetes
type: application
version: 1.4.55
version: 1.4.57

View File

@@ -3,7 +3,7 @@ bot:
repository: harbor.sheldan.dev/sissi
pullPolicy: Always
image: sissi-bot
tag: 1.4.55
tag: 1.4.57
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 5
@@ -20,7 +20,7 @@ restApi:
repository: harbor.sheldan.dev/sissi
pullPolicy: Always
image: sissi-rest-api
tag: 1.4.55
tag: 1.4.57
podAnnotations: {}
podSecurityContext: {}
securityContext: {}
@@ -60,7 +60,7 @@ privateRestApi:
repository: harbor.sheldan.dev/sissi
pullPolicy: Always
image: sissi-private-rest-api
tag: 1.4.55
tag: 1.4.57
podAnnotations: {}
podSecurityContext: {}
securityContext: {}
@@ -91,23 +91,23 @@ templateDeployment:
repository: harbor.sheldan.dev/abstracto
pullPolicy: Always
image: abstracto-template-deployment
tag: 1.5.38
tag: 1.5.44
templateDeploymentData:
repository: harbor.sheldan.dev/sissi
pullPolicy: Always
image: sissi-template-data
tag: 1.4.55
tag: 1.4.57
dbConfigDeployment:
enabled: true
repository: harbor.sheldan.dev/abstracto
pullPolicy: Always
image: abstracto-db-deployment
tag: 1.5.38
tag: 1.5.44
dbConfigDeploymentData:
repository: harbor.sheldan.dev/sissi
pullPolicy: Always
image: sissi-db-data
tag: 1.4.55
tag: 1.4.57
dbCredentials:
host: null
port: null

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi</groupId>
<artifactId>deployment</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi</groupId>
<artifactId>sissi</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -13,13 +13,13 @@
<groupId>dev.sheldan.sissi</groupId>
<artifactId>sissi</artifactId>
<name>Sissi</name>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
<properties>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>17</maven.compiler.source>
<abstracto.version>1.5.38</abstracto.version>
<abstracto.templates.version>1.4.38</abstracto.templates.version>
<abstracto.version>1.5.44</abstracto.version>
<abstracto.templates.version>1.4.44</abstracto.templates.version>
<apache-jena.version>4.9.0</apache-jena.version>
<rssreader.version>3.5.0</rssreader.version>
</properties>
@@ -72,7 +72,7 @@
<connection>scm:git:${project.scm.url}</connection>
<developerConnection>scm:git:${project.scm.url}</developerConnection>
<url>https://github.com/Sheldan/Sissi.git</url>
<tag>HEAD</tag>
<tag>sissi-1.4.58</tag>
</scm>
</project>

View File

@@ -14,13 +14,17 @@ engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password,
with engine.connect() as con:
posts = load_all_starboard_posts(con)
print(posts)
print(f'Loaded {len(posts)}')
enriched_posts = enrich_posts(posts)
print(f'Enriched posts')
import_quotes(enriched_posts, con)
print(f'Done storing quotes')
con.commit()
fix_quote_created(enriched_posts, con)
con.commit()
print('Done.')
sub_posts = chunks = [posts[x:x+100] for x in range(0, len(posts), 100)]
print(f'Loaded {len(posts)} into {len(sub_posts)} partitions')
counter = 0
for sub_post in sub_posts:
print(f'Partition size {len(sub_post)}')
enriched_posts = enrich_posts(sub_post)
print(f'Enriched posts')
import_quotes(enriched_posts, con)
print(f'Done storing quotes')
con.commit()
fix_quote_created(enriched_posts, con)
con.commit()
counter += 1
print(f'Done. {counter}')

View File

@@ -12,7 +12,7 @@ def enrich_posts(posts):
print(f"Loading post {post['message_id']}")
url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}"
message = requests.get(url, headers={'Authorization': token})
time.sleep(5)
time.sleep(1)
if message.status_code == 200:
message_obj = json.loads(message.content)
post['content'] = message_obj['content']

View File

@@ -4,6 +4,7 @@ def import_quotes(posts, con):
for post in posts:
if 'content' not in post:
print(f"Skipping {post['message_id']} because no content, did it fail?")
continue
print(f"Inserting {post['message_id']}")
statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id,
server_id, message_id, text, created)

View File

@@ -2,17 +2,18 @@ from sqlalchemy.sql import text
def load_all_starboard_posts(conn):
squery = text("""select sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
squery = text("""select distinct sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
from starboard_post sp
inner join starboard_post_reaction spr
on sp.id = spr.post_id
and spr.created = (
select spr.created
from starboard_post_reaction spr2
where spr2.post_id = sp.id
order by created limit 1
)
where sp.ignored = false
inner join starboard_post_reaction spr
on sp.id = spr.post_id
and spr.reactor_user_in_server_id = (
select reactor_user_in_server_id
from starboard_post_reaction spr2
where spr2.post_id = sp.id
order by created limit 1
)
where sp.ignored = false
and sp.post_message_id not in (select message_id from quote)
""")
rs = conn.execute(squery)
found_posts = []

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi</groupId>
<artifactId>sissi</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -4,7 +4,7 @@
<parent>
<artifactId>customization-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<artifactId>image-generation-customization-templates</artifactId>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>customization-templates</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>sissi-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>module-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>module-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>sissi-templates</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>module-templates</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>module-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>templates</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>template-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>template-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>template-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>sissi-templates</artifactId>
<groupId>dev.sheldan.sissi.templates</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<artifactId>customization-translations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>sissi-translations</artifactId>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>module-translations</artifactId>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<artifactId>module-translations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<artifactId>sissi-translations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<artifactId>module-translations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<artifactId>module-translations</artifactId>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates</groupId>
<artifactId>templates</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
<artifactId>translation-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
<artifactId>translation-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations.overrides</groupId>
<artifactId>translation-overrides</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -3,7 +3,7 @@
<parent>
<groupId>dev.sheldan.sissi.templates.translations</groupId>
<artifactId>sissi-translations</artifactId>
<version>1.4.56-SNAPSHOT</version>
<version>1.4.58</version>
</parent>
<modelVersion>4.0.0</modelVersion>