Merge pull request #1879 from mfts/feat/index-improvements

feat: improve document processing

Author: mfts

ee/features/security/lib/ratelimit.ts

@@ -11,6 +11,7 @@ export const rateLimiters = {
     redis,
     limiter: Ratelimit.slidingWindow(3, "20 m"),
     prefix: "rl:auth",
+    enableProtection: true,
     analytics: true,
   }),
 
@@ -19,6 +20,7 @@ export const rateLimiters = {
     redis,
     limiter: Ratelimit.slidingWindow(3, "30 m"),
     prefix: "rl:billing",
+    enableProtection: true,
     analytics: true,
   }),
 };

lib/trigger/pdf-to-image-route.ts

@@ -133,6 +133,25 @@ export const convertPdfToImageRoute = task({
         );
 
         if (!response.ok) {
+          const errorData = await response.json().catch(() => ({}));
+
+          // If document was blocked, stop processing entirely
+          if (response.status === 400 && errorData.error?.includes("blocked")) {
+            logger.error("Document blocked", {
+              pageNumber: currentPage,
+              matchedUrl: errorData.matchedUrl,
+              matchedKeyword: errorData.matchedKeyword,
+              payload,
+            });
+
+            updateStatus({
+              progress: 0,
+              text: `Document couldn't be processed`,
+            });
+
+            throw new Error("Document processing blocked");
+          }
+
           throw new Error("Failed to convert page");
         }
 

pages/api/mupdf/convert-page.ts

@@ -1,6 +1,8 @@
 import { NextApiRequest, NextApiResponse } from "next";
 
 import { DocumentPage } from "@prisma/client";
+import { get } from "@vercel/edge-config";
+import { waitUntil } from "@vercel/functions";
 import * as mupdf from "mupdf";
 
 import { putFileServer } from "@/lib/files/put-file-server";
@@ -142,6 +144,43 @@ export default async (req: NextApiRequest, res: NextApiResponse) => {
       return { href: link.getURI(), coords: link.getBounds().join(",") };
     });
 
+    // Check embedded links for blocked keywords
+    if (embeddedLinks.length > 0) {
+      try {
+        const keywords = await get("keywords");
+        if (Array.isArray(keywords) && keywords.length > 0) {
+          for (const link of embeddedLinks) {
+            if (link.href) {
+              const matchedKeyword = keywords.find(
+                (keyword) =>
+                  typeof keyword === "string" && link.href.includes(keyword),
+              );
+
+              if (matchedKeyword) {
+                waitUntil(
+                  log({
+                    message: `Document processing blocked: ${matchedKeyword} \n\n \`Metadata: {teamId: ${teamId}, documentVersionId: ${documentVersionId}, pageNumber: ${pageNumber}}\``,
+                    type: "error",
+                    mention: true,
+                  }),
+                );
+                res.status(400).json({
+                  error: "Document processing blocked",
+                  matchedUrl: link.href,
+                  matchedKeyword: matchedKeyword,
+                  pageNumber: pageNumber,
+                });
+                return;
+              }
+            }
+          }
+        }
+      } catch (error) {
+        // Log error but continue processing if check fails
+        console.log("Failed to check keywords:", error);
+      }
+    }
+
     // Will be updated if we use a reduced scale factor
     let actualScaleFactor = scaleFactor;