Sheldan/Sissi
1
0
Fork 0
mirror of https://github.com/Sheldan/Sissi.git synced 2026-01-26 19:21:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[SIS-xxx] making quote attachment URL column larger

Browse Source 
changing migration script for starboard/quotes
This commit is contained in:
Sheldan
2024-07-13 00:50:45 +02:00
parent af006ee880
commit dda7ed7db8
8 changed files with 51 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/collection.xml Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="tables/tables.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>

11
application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/tables/quote_attachment.xml Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<changeSet author="Sheldan" id="quote_attachment-longer_attachment_url">
<modifyDataType columnName="url"
newDataType="VARCHAR(4096)"
tableName="quote_attachment"/>
</changeSet>
</databaseChangeLog>

6
application/sissi-modules/quotes/src/main/resources/migrations/1.4.57/tables/tables.xml Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="quote_attachment.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>

1
application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml
View File

@@ -4,4 +4,5 @@
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" > xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" >
<include file="1.0.2/collection.xml" relativeToChangelogFile="true"/> <include file="1.0.2/collection.xml" relativeToChangelogFile="true"/>
<include file="1.4.56/collection.xml" relativeToChangelogFile="true"/> <include file="1.4.56/collection.xml" relativeToChangelogFile="true"/>
<include file="1.4.57/collection.xml" relativeToChangelogFile="true"/>
</databaseChangeLog> </databaseChangeLog>

12
python/tools/migrations/quotes-starboard-import/main.py
View File

@@ -14,13 +14,17 @@ engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password,
with engine.connect() as con: with engine.connect() as con:
posts = load_all_starboard_posts(con) posts = load_all_starboard_posts(con)
print(posts) sub_posts = chunks = [posts[x:x+100] for x in range(0, len(posts), 100)]
print(f'Loaded {len(posts)}') print(f'Loaded {len(posts)} into {len(sub_posts)} partitions')
enriched_posts = enrich_posts(posts) counter = 0
for sub_post in sub_posts:
print(f'Partition size {len(sub_post)}')
enriched_posts = enrich_posts(sub_post)
print(f'Enriched posts') print(f'Enriched posts')
import_quotes(enriched_posts, con) import_quotes(enriched_posts, con)
print(f'Done storing quotes') print(f'Done storing quotes')
con.commit() con.commit()
fix_quote_created(enriched_posts, con) fix_quote_created(enriched_posts, con)
con.commit() con.commit()
print('Done.') counter += 1
print(f'Done. {counter}')

2
python/tools/migrations/quotes-starboard-import/post_loader.py
View File

@@ -12,7 +12,7 @@ def enrich_posts(posts):
print(f"Loading post {post['message_id']}") print(f"Loading post {post['message_id']}")
url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}" url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}"
message = requests.get(url, headers={'Authorization': token}) message = requests.get(url, headers={'Authorization': token})
time.sleep(5) time.sleep(0.5)
if message.status_code == 200: if message.status_code == 200:
message_obj = json.loads(message.content) message_obj = json.loads(message.content)
post['content'] = message_obj['content'] post['content'] = message_obj['content']

1
python/tools/migrations/quotes-starboard-import/quote_importer.py
View File

@@ -4,6 +4,7 @@ def import_quotes(posts, con):
for post in posts: for post in posts:
if 'content' not in post: if 'content' not in post:
print(f"Skipping {post['message_id']} because no content, did it fail?") print(f"Skipping {post['message_id']} because no content, did it fail?")
continue
print(f"Inserting {post['message_id']}") print(f"Inserting {post['message_id']}")
statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id, statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id,
server_id, message_id, text, created) server_id, message_id, text, created)

19
python/tools/migrations/quotes-starboard-import/starboard_loader.py
View File

@@ -2,17 +2,18 @@ from sqlalchemy.sql import text
def load_all_starboard_posts(conn): def load_all_starboard_posts(conn):
squery = text("""select sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created squery = text("""select distinct sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created
from starboard_post sp from starboard_post sp
inner join starboard_post_reaction spr inner join starboard_post_reaction spr
on sp.id = spr.post_id on sp.id = spr.post_id
and spr.created = ( and spr.reactor_user_in_server_id = (
select spr.created select reactor_user_in_server_id
from starboard_post_reaction spr2 from starboard_post_reaction spr2
where spr2.post_id = sp.id where spr2.post_id = sp.id
order by created limit 1 order by created limit 1
) )
where sp.ignored = false where sp.ignored = false
and sp.post_message_id not in (select message_id from quote)
""") """)
rs = conn.execute(squery) rs = conn.execute(squery)
found_posts = [] found_posts = []