LLM-friendly docs (llms.txt) and robots.txt (#1972)

* feat: add docusaurus-plugin-llms and robots.txt

Integrate docusaurus-plugin-llms to emit llms.txt and llms-full.txt for
all docs (public networks, private networks, global) with sensible
include order and MDX-friendly cleaning options.

Add static robots.txt (RFC 9309) with Allow rules, common AI crawlers,
and Sitemap pointing at besu.hyperledger.org.

Signed-off-by: bgravenorst <byron.gravenorst@consensys.net>

* fix: move docusaurus-plugin-llms to dependencies for CI

The docs-build workflow installs production dependencies only; the LLMS
plugin is required at docusaurus build time, so it must not be a devDependency.

Signed-off-by: bgravenorst <byron.gravenorst@consensys.net>

* Update docusaurus.config.js

Co-authored-by: Alexandra Carrillo <12214231+alexandratran@users.noreply.github.com>
Signed-off-by: Byron Gravenorst <50852695+bgravenorst@users.noreply.github.com>

---------

Signed-off-by: bgravenorst <byron.gravenorst@consensys.net>
Signed-off-by: Byron Gravenorst <50852695+bgravenorst@users.noreply.github.com>
Co-authored-by: Alexandra Carrillo <12214231+alexandratran@users.noreply.github.com>

Author: bgravenorst

docusaurus.config.js

@@ -329,6 +329,26 @@ const config = {
         ],
       },
     ],
+    [
+      "docusaurus-plugin-llms",
+      {
+        docsDir: "docs",
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: true,
+        title: "Besu documentation",
+        description:
+          "Official Besu documentation: public Ethereum networks, permissioned / private networks, APIs, configuration, and operations.",
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        logLevel: process.env.CI ? "quiet" : "normal",
+        ignoreFiles: ["assets/**"],
+        includeOrder: [
+          "public-networks/**/*",
+          "private-networks/**/*",
+          "global/**/*",
+        ],
+      },
+    ],
   ],
   themes: [
     [

package-lock.json

@@ -36,6 +36,7 @@
     "@docusaurus/theme-common": "^3.10.1",
     "@docusaurus/theme-mermaid": "^3.10.1",
     "@easyops-cn/docusaurus-search-local": "^0.55.0",
+    "docusaurus-plugin-llms": "^0.4.0",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.1.1",
     "open-ask-ai": "^0.7.3",

package.json

@@ -0,0 +1,19 @@
+# https://www.rfc-editor.org/rfc/rfc9309
+# Hyperledger Besu documentation — allow indexing and AI crawlers.
+
+User-agent: *
+Allow: /
+
+User-agent: GPTBot
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+Sitemap: https://besu.hyperledger.org/sitemap.xml