diff --git a/application/config.json.template b/application/config.json.template index 5884522c60..7225704286 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -187,6 +187,7 @@ } ], "fallbackChannelPattern": "java-news-and-changes", + "videoLinkPattern": "http(s)?://www\\.youtube.com.*", "pollIntervalInMinutes": 10 }, "memberCountCategoryPattern": "Info", diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index 1c3371f71a..b18ec6fb1f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -16,6 +16,7 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", required = true) String fallbackChannelPattern, + @JsonProperty(value = "videoLinkPattern", required = true) String videoLinkPattern, @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes) { /** @@ -23,6 +24,8 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) Lis * * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. + * @param videoLinkPattern pattern determining if a link is a video. It is then posted in a way + * to support Discord video embeds. * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are * null @@ -30,5 +33,6 @@ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) Lis public RSSFeedsConfig { Objects.requireNonNull(feeds); Objects.requireNonNull(fallbackChannelPattern); + Objects.requireNonNull(videoLinkPattern); } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/github/GitHubCommand.java b/application/src/main/java/org/togetherjava/tjbot/features/github/GitHubCommand.java index 53a7eed187..b52e79550d 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/github/GitHubCommand.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/github/GitHubCommand.java @@ -4,7 +4,8 @@ import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent; import net.dv8tion.jda.api.interactions.commands.OptionType; import org.kohsuke.github.GHIssue; -import org.kohsuke.github.GHIssueState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.togetherjava.tjbot.features.CommandVisibility; import org.togetherjava.tjbot.features.SlashCommandAdapter; @@ -18,6 +19,7 @@ import java.util.List; import java.util.PriorityQueue; import java.util.Queue; +import java.util.concurrent.CompletableFuture; import java.util.function.ToIntFunction; import java.util.regex.Matcher; import java.util.stream.Stream; @@ -41,11 +43,12 @@ public final class GitHubCommand extends SlashCommandAdapter { }; private static final String TITLE_OPTION = "title"; + private static final Logger logger = LoggerFactory.getLogger(GitHubCommand.class); private final GitHubReference reference; - private Instant lastCacheUpdate; - private List autocompleteGHIssueCache; + private Instant lastCacheUpdate = Instant.EPOCH; + private List autocompleteGHIssueCache = List.of(); /** * Constructs an instance of GitHubCommand. @@ -66,7 +69,14 @@ public GitHubCommand(GitHubReference reference) { getData().addOption(OptionType.STRING, TITLE_OPTION, "Title of the issue you're looking for", true, true); - updateCache(); + CompletableFuture.runAsync(() -> { + try { + updateCache(); + } catch (Exception e) { + logger.error("Unknown error updating the GitHub cache", e); + } + }); + } @Override @@ -111,7 +121,7 @@ public void onAutoComplete(CommandAutoCompleteInteractionEvent event) { event.replyChoiceStrings(choices).queue(); } - if (lastCacheUpdate.isAfter(Instant.now().minus(CACHE_EXPIRES_AFTER))) { + if (isCacheExpired()) { updateCache(); } } @@ -121,12 +131,20 @@ private ToIntFunction suggestionScorer(String title) { return s -> StringDistances.editDistance(title, s.replaceFirst("\\[#\\d+] ", "")); } + private boolean isCacheExpired() { + Instant cacheExpiresAt = lastCacheUpdate.plus(CACHE_EXPIRES_AFTER); + return Instant.now().isAfter(cacheExpiresAt); + } + private void updateCache() { + logger.debug("GitHub Autocomplete cache update started"); + autocompleteGHIssueCache = reference.getRepositories().stream().map(repo -> { try { - return repo.getIssues(GHIssueState.ALL); + return repo.queryIssues().pageSize(1000).list().toList(); } catch (IOException ex) { - throw new UncheckedIOException(ex); + throw new UncheckedIOException("Error fetching issues from repo " + repo.getName(), + ex); } }) .flatMap(List::stream) @@ -135,5 +153,8 @@ private void updateCache() { .toList(); lastCacheUpdate = Instant.now(); + + logger.debug("GitHub autocomplete cache update completed successfully. Cached {} issues.", + autocompleteGHIssueCache.size()); } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/Attachment.java b/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/Attachment.java index 4a7c8ad528..b5fc89644f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/Attachment.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/Attachment.java @@ -2,6 +2,7 @@ import net.dv8tion.jda.api.entities.Message; +import java.util.Locale; import java.util.Optional; import java.util.Set; @@ -10,7 +11,9 @@ record Attachment(String fileName) { Set.of("jpg", "jpeg", "png", "gif", "webp", "tiff", "svg", "apng"); boolean isImage() { - return getFileExtension().map(IMAGE_EXTENSIONS::contains).orElse(false); + return getFileExtension().map(ext -> ext.toLowerCase(Locale.US)) + .map(IMAGE_EXTENSIONS::contains) + .orElse(false); } private Optional getFileExtension() { diff --git a/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/ScamBlocker.java b/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/ScamBlocker.java index 1c37fe8f3f..094d05b5ce 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/ScamBlocker.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/moderation/scam/ScamBlocker.java @@ -48,6 +48,7 @@ import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Listener that receives all sent messages from channels, checks them for scam and takes @@ -247,15 +248,24 @@ private void reportScamMessage(MessageReceivedEvent event, String reportTitle, User author = event.getAuthor(); String avatarOrDefaultUrl = author.getEffectiveAvatarUrl(); - - MessageEmbed embed = - new EmbedBuilder().setDescription(event.getMessage().getContentStripped()) - .setTitle(reportTitle) - .setAuthor(author.getName(), null, avatarOrDefaultUrl) - .setTimestamp(event.getMessage().getTimeCreated()) - .setColor(AMBIENT_COLOR) - .setFooter(author.getId()) - .build(); + String content = event.getMessage().getContentStripped(); + List attachments = event.getMessage().getAttachments(); + + if (!attachments.isEmpty()) { + String attachmentInfo = attachments.stream() + .map(Message.Attachment::getFileName) + .collect(Collectors.joining(", ")); + content += "%s(The message has %d attachment%s: %s)".formatted( + content.isBlank() ? "" : "\n", attachments.size(), + attachments.size() > 1 ? "s " : "", attachmentInfo); + } + MessageEmbed embed = new EmbedBuilder().setDescription(content) + .setTitle(reportTitle) + .setAuthor(author.getName(), null, avatarOrDefaultUrl) + .setTimestamp(event.getMessage().getTimeCreated()) + .setColor(AMBIENT_COLOR) + .setFooter(author.getId()) + .build(); MessageCreateBuilder messageBuilder = new MessageCreateBuilder().setEmbeds(embed); if (!confirmDialog.isEmpty()) { diff --git a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java index c86f305cda..56aea37b74 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java @@ -4,9 +4,9 @@ import com.apptasticsoftware.rssreader.RssReader; import net.dv8tion.jda.api.EmbedBuilder; import net.dv8tion.jda.api.JDA; -import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import net.dv8tion.jda.api.utils.cache.SnowflakeCacheView; +import net.dv8tion.jda.api.utils.messages.MessageCreateData; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.Nullable; import org.jooq.tools.StringUtils; @@ -79,6 +79,7 @@ public final class RSSHandlerRoutine implements Routine { private final RssReader rssReader; private final RSSFeedsConfig config; private final Predicate fallbackChannelPattern; + private final Predicate isVideoLink; private final Map> targetChannelPatterns; private final int interval; private final Database database; @@ -95,6 +96,7 @@ public RSSHandlerRoutine(Config config, Database database) { this.database = database; this.fallbackChannelPattern = Pattern.compile(this.config.fallbackChannelPattern()).asMatchPredicate(); + isVideoLink = Pattern.compile(this.config.videoLinkPattern()).asMatchPredicate(); this.targetChannelPatterns = new HashMap<>(); this.config.feeds().forEach(feed -> { if (feed.targetChannelPattern() != null) { @@ -155,7 +157,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { } rssItems.reversed() .stream() - .filter(shouldItemBePosted.get()) + .filter(shouldItemBePosted.orElseThrow()) .forEachOrdered(item -> postItem(textChannels, item, feedConfig)); } @@ -241,8 +243,8 @@ private Optional getLatestPostDateFromItems(List items, * @param feedConfig the RSS feed configuration */ private void postItem(List textChannels, Item rssItem, RSSFeed feedConfig) { - MessageEmbed embed = constructEmbedMessage(rssItem, feedConfig).build(); - textChannels.forEach(channel -> channel.sendMessageEmbeds(List.of(embed)).queue()); + MessageCreateData message = constructMessage(rssItem, feedConfig); + textChannels.forEach(channel -> channel.sendMessage(message).queue()); } /** @@ -346,13 +348,18 @@ private List getTextChannelsFromFeed(JDA jda, RSSFeed feed) { } /** - * Provides the {@link EmbedBuilder} from an RSS item used for sending RSS posts. + * Provides the message from an RSS item used for sending RSS posts. * * @param item the RSS item to construct the embed message from * @param feedConfig the configuration of the RSS feed - * @return the constructed {@link EmbedBuilder} containing information from the RSS item + * @return the constructed message containing information from the RSS item */ - private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) { + private MessageCreateData constructMessage(Item item, RSSFeed feedConfig) { + if (item.getLink().filter(isVideoLink).isPresent()) { + // Automatic video previews are created on normal messages, not on embeds + return MessageCreateData.fromContent(item.getLink().orElseThrow()); + } + final EmbedBuilder embedBuilder = new EmbedBuilder(); String title = item.getTitle().orElse("No title"); String titleLink = item.getLink().orElse(""); @@ -381,7 +388,7 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) embedBuilder.setDescription("No description"); } - return embedBuilder; + return MessageCreateData.fromEmbeds(embedBuilder.build()); } /** diff --git a/application/src/test/java/org/togetherjava/tjbot/features/moderation/scam/ScamDetectorTest.java b/application/src/test/java/org/togetherjava/tjbot/features/moderation/scam/ScamDetectorTest.java index 4cf982b3f4..52c9b3dffe 100644 --- a/application/src/test/java/org/togetherjava/tjbot/features/moderation/scam/ScamDetectorTest.java +++ b/application/src/test/java/org/togetherjava/tjbot/features/moderation/scam/ScamDetectorTest.java @@ -210,6 +210,23 @@ void ignoresHarmlessAttachments() { assertFalse(isScamResult); } + @Test + @DisplayName("Messages containing suspicious attachments are flagged even if extensions are upper-case (jpg vs JPG)") + void detectsSuspiciousAttachmentsRegardlessOfCase() { + // GIVEN an empty message containing suspicious attachments with mixed case for extensions + String content = ""; + List attachments = + List.of(createImageAttachmentMock("1.JPG"), createImageAttachmentMock("2.JPG"), + createImageAttachmentMock("3.jpg"), createImageAttachmentMock("4.jpg")); + Message message = createMessageMock(content, attachments); + + // WHEN analyzing it + boolean isScamResult = scamDetector.isScam(message); + + // THEN flags it as scam + assertTrue(isScamResult); + } + @Test @DisplayName("Suspicious messages send by trusted users are not flagged") void ignoreTrustedUser() { diff --git a/database/build.gradle b/database/build.gradle index 46b20ac384..79a0587e5f 100644 --- a/database/build.gradle +++ b/database/build.gradle @@ -7,7 +7,7 @@ var sqliteVersion = "3.50.1.0" dependencies { implementation 'com.google.code.findbugs:jsr305:3.0.2' implementation "org.xerial:sqlite-jdbc:${sqliteVersion}" - implementation 'org.flywaydb:flyway-core:11.11.0' + implementation 'org.flywaydb:flyway-core:11.13.0' implementation "org.jooq:jooq:$jooqVersion" implementation project(':utils')