From 7bc04a790645ee71fb239cff3e5af76dd6fa5a40 Mon Sep 17 00:00:00 2001 From: Sheldan <5037282+Sheldan@users.noreply.github.com> Date: Fri, 12 Jul 2024 22:45:17 +0200 Subject: [PATCH] [SIS-xxx] adding sync of starboard posts and quotes, including migration script --- application/sissi-modules/quotes/pom.xml | 8 ++ .../config/QuotesFeatureDefinition.java | 2 +- .../StarboardQuoteSyncFeatureConfig.java | 13 +++ .../StarboardPostCreatedListenerBean.java | 54 +++++++++++ .../StarboardPostDeletedListenerBean.java | 32 +++++++ .../module/quotes/model/database/Quote.java | 12 +-- .../model/database/QuoteAttachment.java | 10 +- .../quotes/repository/QuoteRepository.java | 6 +- .../quotes/service/QuoteServiceBean.java | 47 +++++++--- .../management/QuoteManagementService.java | 94 +++++++++++++++++++ .../migrations/1.4.56/collection.xml | 7 ++ .../migrations/1.4.56/seedData/data.xml | 6 ++ .../migrations/1.4.56/seedData/feature.xml | 10 ++ .../migrations/1.4.56/tables/quote.xml | 16 ++++ .../migrations/1.4.56/tables/tables.xml | 6 ++ .../resources/migrations/quotes-changeLog.xml | 1 + .../src/main/resources/quotes.properties | 3 + .../quotes-starboard-import/main.py | 26 +++++ .../quotes-starboard-import/post_loader.py | 32 +++++++ .../quotes-starboard-import/quote_importer.py | 25 +++++ .../starboard_loader.py | 29 ++++++ 21 files changed, 412 insertions(+), 27 deletions(-) create mode 100644 application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/StarboardQuoteSyncFeatureConfig.java create mode 100644 application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostCreatedListenerBean.java create mode 100644 application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostDeletedListenerBean.java create mode 100644 application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/management/QuoteManagementService.java create mode 100644 application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/collection.xml create mode 100644 application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/data.xml create mode 100644 application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/feature.xml create mode 100644 application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/quote.xml create mode 100644 application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/tables.xml create mode 100644 python/tools/migrations/quotes-starboard-import/main.py create mode 100644 python/tools/migrations/quotes-starboard-import/post_loader.py create mode 100644 python/tools/migrations/quotes-starboard-import/quote_importer.py create mode 100644 python/tools/migrations/quotes-starboard-import/starboard_loader.py diff --git a/application/sissi-modules/quotes/pom.xml b/application/sissi-modules/quotes/pom.xml index f676314b..b8de9fbc 100644 --- a/application/sissi-modules/quotes/pom.xml +++ b/application/sissi-modules/quotes/pom.xml @@ -10,6 +10,14 @@ dev.sheldan.sissi.application.module quotes + + + dev.sheldan.abstracto.modules + starboard-int + ${abstracto.version} + + + diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/QuotesFeatureDefinition.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/QuotesFeatureDefinition.java index 29592c8f..a578b38e 100644 --- a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/QuotesFeatureDefinition.java +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/QuotesFeatureDefinition.java @@ -5,7 +5,7 @@ import lombok.Getter; @Getter public enum QuotesFeatureDefinition implements FeatureDefinition { - QUOTES("quotes"); + QUOTES("quotes"), STARBOARD_QUOTE_SYNC("starboardQuoteSync"); private String key; diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/StarboardQuoteSyncFeatureConfig.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/StarboardQuoteSyncFeatureConfig.java new file mode 100644 index 00000000..b4cd0e00 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/config/StarboardQuoteSyncFeatureConfig.java @@ -0,0 +1,13 @@ +package dev.sheldan.sissi.module.quotes.config; + +import dev.sheldan.abstracto.core.config.FeatureConfig; +import dev.sheldan.abstracto.core.config.FeatureDefinition; +import org.springframework.stereotype.Component; + +@Component +public class StarboardQuoteSyncFeatureConfig implements FeatureConfig { + @Override + public FeatureDefinition getFeature() { + return QuotesFeatureDefinition.STARBOARD_QUOTE_SYNC; + } +} diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostCreatedListenerBean.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostCreatedListenerBean.java new file mode 100644 index 00000000..62e7ed42 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostCreatedListenerBean.java @@ -0,0 +1,54 @@ +package dev.sheldan.sissi.module.quotes.listener; + +import dev.sheldan.abstracto.core.config.FeatureDefinition; +import dev.sheldan.abstracto.core.listener.DefaultListenerResult; +import dev.sheldan.abstracto.core.service.MessageService; +import dev.sheldan.abstracto.starboard.listener.StarboardPostCreatedListener; +import dev.sheldan.abstracto.starboard.model.StarboardPostCreatedModel; +import dev.sheldan.sissi.module.quotes.config.QuotesFeatureDefinition; +import dev.sheldan.sissi.module.quotes.service.QuoteServiceBean; +import lombok.extern.slf4j.Slf4j; +import net.dv8tion.jda.api.entities.Message; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Component +public class StarboardPostCreatedListenerBean implements StarboardPostCreatedListener { + + @Autowired + private MessageService messageService; + + @Autowired + private QuoteServiceBean quoteServiceBean; + + @Autowired + private StarboardPostCreatedListenerBean self; + + @Override + public DefaultListenerResult execute(StarboardPostCreatedModel model) { + Long serverId = model.getServerId(); + Long starboardPostId = model.getStarboardPostId(); + messageService.loadMessage(serverId, model.getStarredMessage().getChannelId(), model.getStarredMessage().getMessageId()) + .thenAccept(message -> self.storeQuote(message, model)) + .exceptionally(throwable -> { + log.error("Failed to persist quote for starboard post {} in server {}.", starboardPostId, serverId, throwable); + return null; + }); + return DefaultListenerResult.PROCESSED; + } + + @Transactional + public void storeQuote(Message message, StarboardPostCreatedModel model) { + log.info("Creating quote from starboard post {} in server {} from user {} because of user {}.", model.getStarboardPostId(), model.getServerId(), + model.getStarredUser().getUserId(), model.getLastStarrer().getUserId()); + quoteServiceBean.createQuote(model.getStarredUser(), model.getLastStarrer(), message); + } + + @Override + public FeatureDefinition getFeature() { + return QuotesFeatureDefinition.STARBOARD_QUOTE_SYNC; + } + +} diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostDeletedListenerBean.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostDeletedListenerBean.java new file mode 100644 index 00000000..77e5936b --- /dev/null +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/listener/StarboardPostDeletedListenerBean.java @@ -0,0 +1,32 @@ +package dev.sheldan.sissi.module.quotes.listener; + +import dev.sheldan.abstracto.core.config.FeatureDefinition; +import dev.sheldan.abstracto.core.listener.DefaultListenerResult; +import dev.sheldan.abstracto.starboard.listener.StarboardPostDeletedListener; +import dev.sheldan.abstracto.starboard.model.StarboardPostDeletedModel; +import dev.sheldan.sissi.module.quotes.config.QuotesFeatureDefinition; +import dev.sheldan.sissi.module.quotes.service.QuoteServiceBean; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class StarboardPostDeletedListenerBean implements StarboardPostDeletedListener { + + @Autowired + private QuoteServiceBean quoteServiceBean; + + @Override + public DefaultListenerResult execute(StarboardPostDeletedModel model) { + log.info("Handling delete of starboard post {}, causing the quote of message {} in server {} to be deleted.", model.getStarboardPostId(), model.getStarredMessage().getMessageId(), model.getServerId()); + quoteServiceBean.deleteByMessageId(model.getStarredMessage().getMessageId()); + return DefaultListenerResult.PROCESSED; + } + + @Override + public FeatureDefinition getFeature() { + return QuotesFeatureDefinition.STARBOARD_QUOTE_SYNC; + } + +} diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/Quote.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/Quote.java index 22398a73..afae1849 100644 --- a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/Quote.java +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/Quote.java @@ -21,12 +21,12 @@ import java.util.List; @EqualsAndHashCode public class Quote { - @EmbeddedId - @Getter - private ServerSpecificId id; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; @ManyToOne(cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch = FetchType.LAZY) - @MapsId("serverId") @JoinColumn(name = "server_id", nullable = false) private AServer server; @@ -58,9 +58,9 @@ public class Quote { @Column(name = "text") private String text; - @Column(name = "created") + @Column(name = "created", insertable = false, updatable = false) private Instant created; - @Column(name = "updated") + @Column(name = "updated", insertable = false, updatable = false) private Instant updated; } diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/QuoteAttachment.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/QuoteAttachment.java index 40ba4b6c..000b7ffb 100644 --- a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/QuoteAttachment.java +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/model/database/QuoteAttachment.java @@ -1,5 +1,6 @@ package dev.sheldan.sissi.module.quotes.model.database; +import dev.sheldan.abstracto.core.models.database.AServer; import lombok.*; import jakarta.persistence.*; @@ -16,7 +17,7 @@ public class QuoteAttachment { @Id @Getter - @Setter + @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", nullable = false) private Long id; @@ -25,8 +26,7 @@ public class QuoteAttachment { @ManyToOne(fetch = FetchType.LAZY) @JoinColumns( { - @JoinColumn(updatable = false, insertable = false, name = "quote_id", referencedColumnName = "id"), - @JoinColumn(updatable = false, insertable = false, name = "server_id", referencedColumnName = "server_id") + @JoinColumn(updatable = false, name = "quote_id", referencedColumnName = "id") }) private Quote quote; @@ -35,6 +35,10 @@ public class QuoteAttachment { @Column(name = "url", nullable = false) private String url; + @ManyToOne(cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch = FetchType.LAZY) + @JoinColumn(name = "server_id", nullable = false) + private AServer server; + @Getter @Setter @Column(name = "is_image", nullable = false) diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/repository/QuoteRepository.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/repository/QuoteRepository.java index 6c9eb27b..c9a0975d 100644 --- a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/repository/QuoteRepository.java +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/repository/QuoteRepository.java @@ -1,6 +1,5 @@ package dev.sheldan.sissi.module.quotes.repository; -import dev.sheldan.abstracto.core.models.ServerSpecificId; import dev.sheldan.abstracto.core.models.database.AServer; import dev.sheldan.abstracto.core.models.database.AUserInAServer; import dev.sheldan.sissi.module.quotes.model.database.Quote; @@ -8,13 +7,16 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import java.util.List; +import java.util.Optional; @Repository -public interface QuoteRepository extends JpaRepository { +public interface QuoteRepository extends JpaRepository { List findByTextContainingAndServer(String text, AServer server); List findByTextContainingAndServerAndAuthor(String text, AServer server, AUserInAServer author); List findByServer(AServer server); List findByAuthor(AUserInAServer author); Long countByAuthor(AUserInAServer author); Long countByAdder(AUserInAServer adder); + + Optional findByMessageId(Long messageId); } diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/QuoteServiceBean.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/QuoteServiceBean.java index 28193bc9..1ef1a04f 100644 --- a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/QuoteServiceBean.java +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/QuoteServiceBean.java @@ -1,7 +1,7 @@ package dev.sheldan.sissi.module.quotes.service; import dev.sheldan.abstracto.core.models.ServerChannelMessage; -import dev.sheldan.abstracto.core.models.ServerSpecificId; +import dev.sheldan.abstracto.core.models.ServerUser; import dev.sheldan.abstracto.core.models.database.AServer; import dev.sheldan.abstracto.core.models.database.AUserInAServer; import dev.sheldan.abstracto.core.service.ChannelService; @@ -16,11 +16,12 @@ import dev.sheldan.sissi.module.quotes.model.command.QuoteResponseModel; import dev.sheldan.sissi.module.quotes.model.command.QuoteStatsModel; import dev.sheldan.sissi.module.quotes.model.database.Quote; import dev.sheldan.sissi.module.quotes.model.database.QuoteAttachment; -import dev.sheldan.sissi.module.quotes.repository.QuoteRepository; +import dev.sheldan.sissi.module.quotes.service.management.QuoteManagementService; import lombok.extern.slf4j.Slf4j; import net.dv8tion.jda.api.entities.*; import net.dv8tion.jda.api.entities.channel.Channel; import net.dv8tion.jda.api.entities.channel.middleman.GuildMessageChannel; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -35,9 +36,6 @@ import java.util.stream.Collectors; @Slf4j public class QuoteServiceBean { - @Autowired - private QuoteRepository quoteRepository; - @Autowired private MemberService memberService; @@ -56,11 +54,14 @@ public class QuoteServiceBean { @Autowired private ChannelService channelService; + @Autowired + private QuoteManagementService quoteManagementService; + private static final String QUOTE_RESPONSE_TEMPLATE_KEY = "quote_response"; public Optional getRandomQuoteForMember(AUserInAServer aUserInAServer) { // not nice, but good enough for now - List allQuotes = quoteRepository.findByAuthor(aUserInAServer); + List allQuotes = quoteManagementService.getFromAuthor(aUserInAServer); if(allQuotes.isEmpty()) { return Optional.empty(); } @@ -69,7 +70,7 @@ public class QuoteServiceBean { public Optional getRandomQuote(AServer server) { // not nice, but good enough for now - List allQuotes = quoteRepository.findByServer(server); + List allQuotes = quoteManagementService.getFromServer(server); if(allQuotes.isEmpty()) { return Optional.empty(); } @@ -79,7 +80,7 @@ public class QuoteServiceBean { public void deleteQuote(Long quoteId, AServer server) { Optional existingQuote = getQuote(quoteId, server); if(existingQuote.isPresent()) { - quoteRepository.delete(existingQuote.get()); + quoteManagementService.deleteQuote(existingQuote.get()); log.info("Deleting quote with id {} in server {}.", quoteId, server.getId()); } else { throw new QuoteNotFoundException(); @@ -87,9 +88,8 @@ public class QuoteServiceBean { } public Optional getQuote(Long quoteId, AServer server) { - ServerSpecificId id = new ServerSpecificId(server.getId(), quoteId); log.info("Loading quote with id {} in server {}.", quoteId, server.getId()); - return quoteRepository.findById(id); + return quoteManagementService.getQuote(quoteId); } public CompletableFuture renderQuoteToMessageToSend(Quote quote) { @@ -118,7 +118,7 @@ public class QuoteServiceBean { .builder() .quoteContent(quote.getText()) .imageAttachmentURLs(imageAttachments) - .quoteId(quote.getId().getId()) + .quoteId(quote.getId()) .fileAttachmentURLs(fileAttachments) .creationDate(quote.getCreated()) .quotedMessage(quotedMessage); @@ -209,7 +209,7 @@ public class QuoteServiceBean { } public Optional searchQuote(String query, AServer server) { - List foundQuotes = quoteRepository.findByTextContainingAndServer(query, server); + List foundQuotes = quoteManagementService.getQuotesWithTextInServer(query, server); if(foundQuotes.isEmpty()) { return Optional.empty(); } @@ -224,7 +224,7 @@ public class QuoteServiceBean { public Optional searchQuote(String query, AServer server, Member targetMember) { AUserInAServer author = userInServerManagementService.loadOrCreateUser(targetMember); - List foundQuotes = quoteRepository.findByTextContainingAndServerAndAuthor(query, server, author); + List foundQuotes = quoteManagementService.getQuotesWithTextInServerFromAuthor(query, server, author); if(foundQuotes.isEmpty()) { return Optional.empty(); } @@ -242,8 +242,8 @@ public class QuoteServiceBean { return getQuoteStats(user, member); } public QuoteStatsModel getQuoteStats(AUserInAServer user, Member member) { - Long authored = quoteRepository.countByAuthor(user); - Long added = quoteRepository.countByAdder(user); + Long authored = quoteManagementService.getAmountOfQuotesOfAuthor(user); + Long added = quoteManagementService.getAmountOfQuotesOfAdder(user); return QuoteStatsModel .builder() .quoteCount(added) @@ -253,4 +253,21 @@ public class QuoteServiceBean { .serverId(user.getServerReference().getId()) .build(); } + + public Quote createQuote(ServerUser authorUser, ServerUser adderUser, Message quoteMessage) { + AUserInAServer author = userInServerManagementService.loadOrCreateUser(authorUser); + AUserInAServer adder = userInServerManagementService.loadOrCreateUser(adderUser); + List> attachments = quoteMessage + .getAttachments() + .stream() + .map(attachment -> Pair.of(attachment.getProxyUrl(), attachment.isImage())) + .toList(); + return quoteManagementService.createQuote(author, adder, quoteMessage.getContentDisplay(), ServerChannelMessage.fromMessage(quoteMessage), attachments); + } + + public void deleteByMessageId(Long messageId) { + Quote quote = quoteManagementService.findByMessage(messageId).orElseThrow(QuoteNotFoundException::new); + log.info("Deleting quote {} in server {}.", quote.getId(), quote.getServer().getId()); + quoteManagementService.deleteQuote(quote); + } } diff --git a/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/management/QuoteManagementService.java b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/management/QuoteManagementService.java new file mode 100644 index 00000000..17535dc2 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/java/dev/sheldan/sissi/module/quotes/service/management/QuoteManagementService.java @@ -0,0 +1,94 @@ +package dev.sheldan.sissi.module.quotes.service.management; + +import dev.sheldan.abstracto.core.models.ServerChannelMessage; +import dev.sheldan.abstracto.core.models.database.AChannel; +import dev.sheldan.abstracto.core.models.database.AServer; +import dev.sheldan.abstracto.core.models.database.AUserInAServer; +import dev.sheldan.abstracto.core.service.management.ChannelManagementService; +import dev.sheldan.sissi.module.quotes.model.database.Quote; +import dev.sheldan.sissi.module.quotes.model.database.QuoteAttachment; +import dev.sheldan.sissi.module.quotes.repository.QuoteRepository; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; + +@Component +@Slf4j +public class QuoteManagementService { + + @Autowired + private QuoteRepository quoteRepository; + + @Autowired + private ChannelManagementService channelManagementService; + + public Quote createQuote(AUserInAServer author, AUserInAServer adder, String messageText, ServerChannelMessage quotedMessage, List> attachments) { + AChannel channel = channelManagementService.loadChannel(quotedMessage.getChannelId()); + + Quote quote = Quote + .builder() + .adder(adder) + .author(author) + .text(messageText) + .messageId(quotedMessage.getMessageId()) + .server(adder.getServerReference()) + .sourceChannel(channel) + .build(); + List quoteAttachments = attachments + .stream() + .map(stringBooleanPair -> QuoteAttachment + .builder() + .url(stringBooleanPair.getLeft()) + .quote(quote) + .server(adder.getServerReference()) + .isImage(stringBooleanPair.getRight()) + .build()) + .toList(); + + log.info("Creating quote from {} added by {} in server {}.", author.getUserReference().getId(), adder.getUserReference().getId(), author.getServerReference().getId()); + + quote.setAttachments(quoteAttachments); + + return quoteRepository.save(quote); + } + + public List getFromAuthor(AUserInAServer author) { + return quoteRepository.findByAuthor(author); + } + + public List getFromServer(AServer aServer) { + return quoteRepository.findByServer(aServer); + } + + public void deleteQuote(Quote quote) { + quoteRepository.delete(quote); + } + + public Optional getQuote(Long quoteId) { + return quoteRepository.findById(quoteId); + } + + public List getQuotesWithTextInServer(String text, AServer server) { + return quoteRepository.findByTextContainingAndServer(text, server); + } + + public List getQuotesWithTextInServerFromAuthor(String text, AServer server, AUserInAServer aUserInAServer) { + return quoteRepository.findByTextContainingAndServerAndAuthor(text, server, aUserInAServer); + } + + public Long getAmountOfQuotesOfAuthor(AUserInAServer aUserInAServer) { + return quoteRepository.countByAuthor(aUserInAServer); + } + + public Long getAmountOfQuotesOfAdder(AUserInAServer aUserInAServer) { + return quoteRepository.countByAdder(aUserInAServer); + } + + public Optional findByMessage(Long messageId) { + return quoteRepository.findByMessageId(messageId); + } +} diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/collection.xml b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/collection.xml new file mode 100644 index 00000000..a390f38e --- /dev/null +++ b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/collection.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/data.xml b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/data.xml new file mode 100644 index 00000000..f4ad8387 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/data.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/feature.xml b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/feature.xml new file mode 100644 index 00000000..1bed6367 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/seedData/feature.xml @@ -0,0 +1,10 @@ + + + + + + + + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/quote.xml b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/quote.xml new file mode 100644 index 00000000..74dff252 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/quote.xml @@ -0,0 +1,16 @@ + + + + + DROP TRIGGER IF EXISTS quote_update_trigger ON quote; + CREATE TRIGGER quote_update_trigger BEFORE UPDATE ON quote FOR EACH ROW EXECUTE PROCEDURE update_trigger_procedure(); + + + DROP TRIGGER IF EXISTS quote_insert_trigger ON quote; + CREATE TRIGGER quote_insert_trigger BEFORE INSERT ON quote FOR EACH ROW EXECUTE PROCEDURE insert_trigger_procedure(); + + + + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/tables.xml b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/tables.xml new file mode 100644 index 00000000..9f4b81d8 --- /dev/null +++ b/application/sissi-modules/quotes/src/main/resources/migrations/1.4.56/tables/tables.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml b/application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml index 6f8a37a8..c516c477 100644 --- a/application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml +++ b/application/sissi-modules/quotes/src/main/resources/migrations/quotes-changeLog.xml @@ -3,4 +3,5 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" > + \ No newline at end of file diff --git a/application/sissi-modules/quotes/src/main/resources/quotes.properties b/application/sissi-modules/quotes/src/main/resources/quotes.properties index 3f1bd5fd..7d3805a6 100644 --- a/application/sissi-modules/quotes/src/main/resources/quotes.properties +++ b/application/sissi-modules/quotes/src/main/resources/quotes.properties @@ -1,2 +1,5 @@ abstracto.featureFlags.quotes.featureName=quotes abstracto.featureFlags.quotes.enabled=false + +abstracto.featureFlags.starboardQuoteSync.featureName=starboardQuoteSync +abstracto.featureFlags.starboardQuoteSync.enabled=false diff --git a/python/tools/migrations/quotes-starboard-import/main.py b/python/tools/migrations/quotes-starboard-import/main.py new file mode 100644 index 00000000..9a6b0afc --- /dev/null +++ b/python/tools/migrations/quotes-starboard-import/main.py @@ -0,0 +1,26 @@ +import sqlalchemy as db +import os +from starboard_loader import load_all_starboard_posts +from post_loader import enrich_posts +from quote_importer import import_quotes, fix_quote_created + +db_host = os.getenv('DB_HOST') +db_port = os.getenv('DB_PORT') +db_database = os.getenv('DB_NAME') +db_user = os.getenv('DB_USER') +db_password = os.getenv('DB_PASS') + +engine = db.create_engine('postgresql://%s:%s@%s:%s/%s' % (db_user, db_password, db_host, db_port, db_database)) + +with engine.connect() as con: + posts = load_all_starboard_posts(con) + print(posts) + print(f'Loaded {len(posts)}') + enriched_posts = enrich_posts(posts) + print(f'Enriched posts') + import_quotes(enriched_posts, con) + print(f'Done storing quotes') + con.commit() + fix_quote_created(enriched_posts, con) + con.commit() + print('Done.') \ No newline at end of file diff --git a/python/tools/migrations/quotes-starboard-import/post_loader.py b/python/tools/migrations/quotes-starboard-import/post_loader.py new file mode 100644 index 00000000..ca41cd56 --- /dev/null +++ b/python/tools/migrations/quotes-starboard-import/post_loader.py @@ -0,0 +1,32 @@ +import requests +import os +import json +import time + +token = os.getenv('TOKEN') + +image_extension = ["jpg", "jpeg", "png", "gif", "webp", "tiff", "svg", "apng"] + +def enrich_posts(posts): + for post in posts: + print(f"Loading post {post['message_id']}") + url = f"https://discord.com/api/v10/channels/{post['channel_id']}/messages/{post['message_id']}" + message = requests.get(url, headers={'Authorization': token}) + time.sleep(5) + if message.status_code == 200: + message_obj = json.loads(message.content) + post['content'] = message_obj['content'] + attachments = [] + attachment_objs = message_obj['attachments'] + if len(attachment_objs) > 0: + for attachment in attachment_objs: + extension = attachment['filename'][attachment['filename'].rfind('.') + 1] + attachment = { + 'url': attachment['proxy_url'], + 'is_image': extension.lower() in image_extension + } + attachments.append(attachment) + post['attachments'] = attachments + else: + print(f"{post['message_id']}: Didnt find post {url}: {message.status_code}") + return posts diff --git a/python/tools/migrations/quotes-starboard-import/quote_importer.py b/python/tools/migrations/quotes-starboard-import/quote_importer.py new file mode 100644 index 00000000..c429a894 --- /dev/null +++ b/python/tools/migrations/quotes-starboard-import/quote_importer.py @@ -0,0 +1,25 @@ +from sqlalchemy.sql import text + +def import_quotes(posts, con): + for post in posts: + if 'content' not in post: + print(f"Skipping {post['message_id']} because no content, did it fail?") + print(f"Inserting {post['message_id']}") + statement = text("""INSERT INTO quote(author_user_in_server_id, adder_user_in_server_id, source_channel_id, + server_id, message_id, text, created) + VALUES(:author_id, :adder_id, :channel_id, :server_id, :message_id, :content, :created) returning id""") + quote_id = con.execute(statement, {'author_id': post['author_id'], 'adder_id': post['adder_id'], 'channel_id': post['channel_id'], 'server_id': post['server_id'], + 'message_id': post['message_id'], 'content': post['content'], 'created': post['created']}).fetchone()[0] + print(f'Created quote {quote_id}') + for attachment in post['attachments']: + statement = text("""INSERT INTO quote_attachment(quote_id, server_id, url, is_image) + VALUES(:quote_id, :server_id, :url, :is_image)""") + con.execute(statement, {'quote_id': quote_id, 'server_id': post['server_id'], 'url': attachment['url'], 'is_image': attachment['is_image']}) + post['quote_id'] = quote_id + +# the insert trigger always updated created, we have to re-do it (will be changed, but not for now) +def fix_quote_created(posts, con): + for post in posts: + if 'quote_id' in post: + statement = text("""update quote set created = :created where id = :quote_id""") + con.execute(statement, {'created': post['created'], 'quote_id': post['quote_id']}) diff --git a/python/tools/migrations/quotes-starboard-import/starboard_loader.py b/python/tools/migrations/quotes-starboard-import/starboard_loader.py new file mode 100644 index 00000000..4e6212b7 --- /dev/null +++ b/python/tools/migrations/quotes-starboard-import/starboard_loader.py @@ -0,0 +1,29 @@ +from sqlalchemy.sql import text + + +def load_all_starboard_posts(conn): + squery = text("""select sp.id, sp.author_user_in_server_id, sp.source_channel_id, sp.server_id, sp.post_message_id, spr.reactor_user_in_server_id, sp.created +from starboard_post sp +inner join starboard_post_reaction spr +on sp.id = spr.post_id +and spr.created = ( +select spr.created +from starboard_post_reaction spr2 +where spr2.post_id = sp.id +order by created limit 1 + ) + where sp.ignored = false + """) + rs = conn.execute(squery) + found_posts = [] + for post in rs: + found_posts.append({ + 'post_id': post[0], + 'channel_id': post[2], + 'message_id': post[4], + 'adder_id': post[5], + 'author_id': post[1], + 'server_id': post[3], + 'created': post[6] + }) + return found_posts