feat(check-html-links): add external link validation

Author: jorgecasar

.changeset/great-bobcats-destroy.md

@@ -0,0 +1,5 @@
+---
+'check-html-links': minor
+---
+
+add external links validation using the flag `--validate-externals`.

package.json

@@ -81,7 +81,7 @@
     "husky": "^4.3.7",
     "lint-staged": "^10.5.3",
     "mocha": "^9.1.3",
-    "node-fetch": "^2.6.7",
+    "node-fetch": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "onchange": "^7.1.0",
     "prettier": "^2.5.1",

packages/check-html-links/README.md

@@ -14,7 +14,7 @@ npm i -D check-html-links
 npx check-html-links _site
 ```
 
-For docs please see our homepage [https://rocket.modern-web.dev/docs/tools/check-html-links/](https://rocket.modern-web.dev/docs/tools/check-html-links/).
+For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
 
 ## Comparison
 

packages/check-html-links/package.json

@@ -37,6 +37,7 @@
     "command-line-args": "^5.1.1",
     "glob": "^7.0.0",
     "minimatch": "^3.0.4",
+    "node-fetch": "^3.0.0",
     "sax-wasm": "^2.0.0",
     "slash": "^4.0.0"
   },

packages/check-html-links/src/CheckHtmlLinksCli.js

@@ -18,7 +18,8 @@ export class CheckHtmlLinksCli {
     const mainDefinitions = [
       { name: 'ignore-link-pattern', type: String, multiple: true },
       { name: 'root-dir', type: String, defaultOption: true },
-      { name: 'continue-on-error', type: Boolean, defaultOption: false },
+      { name: 'continue-on-error', type: Boolean },
+      { name: 'validate-externals', type: Boolean },
     ];
     const options = commandLineArgs(mainDefinitions, {
       stopAtFirstUnknown: true,
@@ -29,6 +30,7 @@ export class CheckHtmlLinksCli {
       continueOnError: options['continue-on-error'],
       rootDir: options['root-dir'],
       ignoreLinkPatterns: options['ignore-link-pattern'],
+      validateExternals: options['validate-externals'],
     };
   }
 
@@ -43,7 +45,7 @@ export class CheckHtmlLinksCli {
   }
 
   async run() {
-    const { ignoreLinkPatterns, rootDir: userRootDir } = this.options;
+    const { ignoreLinkPatterns, rootDir: userRootDir, validateExternals } = this.options;
     const rootDir = userRootDir ? path.resolve(userRootDir) : process.cwd();
     const performanceStart = process.hrtime();
 
@@ -56,7 +58,10 @@ export class CheckHtmlLinksCli {
         : `🔥 Found a total of ${chalk.green.bold(files.length)} files to check!`;
     console.log(filesOutput);
 
-    const { errors, numberLinks } = await validateFiles(files, rootDir, { ignoreLinkPatterns });
+    const { errors, numberLinks } = await validateFiles(files, rootDir, {
+      ignoreLinkPatterns,
+      validateExternals,
+    });
 
     console.log(`🔗 Found a total of ${chalk.green.bold(numberLinks)} links to validate!\n`);
 

packages/check-html-links/src/checkLinks.js

@@ -0,0 +1,59 @@
+import fetch from 'node-fetch';
+
+/**
+ * @type {Map<string,boolean>}
+ */
+const resultsMap = new Map();
+
+/**
+ *
+ * @param {string} url
+ * @param {boolean} result
+ * @returns {boolean}
+ */
+const memorizeCheckup = (url, result) => {
+  resultsMap.set(url, result);
+  return result;
+};
+
+/**
+ *
+ * @param {string} url
+ * @param {string} method
+ * @returns
+ */
+const fetchWrap = async (url, method = 'GET') =>
+  fetch(url, { method })
+    .then(response => response.ok)
+    .catch(() => false);
+
+/**
+ *
+ * @param {string} url
+ * @returns {Promise<boolean>}
+ */
+const fetchHead = async url => fetchWrap(url, 'HEAD');
+
+/**
+ *
+ * @param {string} url - URL object to check
+ * @returns {Promise<boolean>} true if url is alive or false if not
+ */
+const checkUrl = async url =>
+  (fetchHead(url) || fetchWrap(url)).then(result => memorizeCheckup(url, result));
+
+/**
+ *
+ * @param {string} link - link string to check
+ * @returns {Promise<boolean>}
+ */
+export const checkLink = async link => {
+  const url = link.startsWith('//') ? `https:${link}` : link;
+  return resultsMap.get(url) ?? checkUrl(url);
+};
+/**
+ * Check an array of links and return an object with
+ *
+ * @param {string[]} links Links to check
+ */
+export const checkLinks = async links => Promise.all(links.map(checkLink));

packages/check-html-links/src/validateFolder.js

@@ -3,10 +3,10 @@ import fs from 'fs';
 import saxWasm from 'sax-wasm';
 import minimatch from 'minimatch';
 import { createRequire } from 'module';
-
-import { listFiles } from './listFiles.js';
 import path from 'path';
 import slash from 'slash';
+import { listFiles } from './listFiles.js';
+import { checkLinks } from './checkLinks.js';
 
 /** @typedef {import('../types/main').Link} Link */
 /** @typedef {import('../types/main').LocalFile} LocalFile */
@@ -28,6 +28,9 @@ const parserIds = new SAXParser(SaxEventType.Attribute, streamOptions);
 /** @type {Error[]} */
 let checkLocalFiles = [];
 
+/** @type {Error[]} */
+let checkExternalLinks = [];
+
 /** @type {Error[]} */
 let errors = [];
 
@@ -151,6 +154,26 @@ function addLocalFile(filePath, anchor, usageObj) {
   }
 }
 
+/**
+ * @param {string} filePath
+ * @param {Usage} usageObj
+ */
+function addExternalLink(filePath, usageObj) {
+  const foundIndex = checkExternalLinks.findIndex(item => {
+    return item.filePath === filePath;
+  });
+
+  if (foundIndex === -1) {
+    checkExternalLinks.push({
+      filePath,
+      onlyAnchorMissing: false,
+      usage: [usageObj],
+    });
+  } else {
+    checkExternalLinks[foundIndex].usage.push(usageObj);
+  }
+}
+
 /**
  * @param {string} inValue
  */
@@ -231,6 +254,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     } else if (value.startsWith('//') || value.startsWith('http')) {
       // TODO: handle external urls
       // external url - we do not handle that (yet)
+      addExternalLink(htmlFilePath, usageObj);
     } else if (value.startsWith('/')) {
       const filePath = path.join(rootDir, valueFile);
       addLocalFile(filePath, anchor, usageObj);
@@ -244,7 +268,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     }
   }
 
-  return { checkLocalFiles: [...checkLocalFiles] };
+  return { checkLocalFiles: [...checkLocalFiles], checkExternalLinks: [...checkExternalLinks] };
 }
 
 /**
@@ -283,6 +307,22 @@ async function validateLocalFiles(checkLocalFiles) {
   return errors;
 }
 
+/**
+ *
+ * @param {Error[]} checkExternalLinks
+ */
+async function validateExternalLinks(checkExternalLinks) {
+  for await (const localFileObj of checkExternalLinks) {
+    const links = localFileObj.usage.map(usage => usage.value);
+    const results = await checkLinks(links);
+    localFileObj.usage = localFileObj.usage.filter((link, index) => !results[index]);
+    if (localFileObj.usage.length > 0) {
+      errors.push(localFileObj);
+    }
+  }
+  return errors;
+}
+
 /**
  * @param {string[]} files
  * @param {string} rootDir
@@ -294,6 +334,7 @@ export async function validateFiles(files, rootDir, opts) {
 
   errors = [];
   checkLocalFiles = [];
+  checkExternalLinks = [];
   idCache = new Map();
   let numberLinks = 0;
 
@@ -312,6 +353,9 @@ export async function validateFiles(files, rootDir, opts) {
     await resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage });
   }
   await validateLocalFiles(checkLocalFiles);
+  if (opts?.validateExternals) {
+    await validateExternalLinks(checkExternalLinks);
+  }
 
   return { errors: errors, numberLinks: numberLinks };
 }

packages/check-html-links/test-node/fixtures/external-link/index.html

@@ -1,5 +1,9 @@
-<!-- ignore known subsystems -->
-<a href="/docs/"></a>
-<a href="/developer/getting-started.html#js"></a>
-<a href="/developer/language-guides/"></a>
-<a href="/developer/javascript.html"></a>
+<!-- valid -->
+<a href="//rocket.modern-web.dev/"></a>
+<a href="http://rocket.modern-web.dev/"></a>
+<a href="https://rocket.modern-web.dev/"></a>
+
+<!-- invalid -->
+<a href="//rocket.modern-web.dev/unexists-page/"></a>
+<a href="http://rocket.modern-web.dev/unexists-page/"></a>
+<a href="https://rocket.modern-web.dev/unexists-page/"></a>

packages/check-html-links/test-node/fixtures/internal-link/index.html

@@ -5,8 +5,5 @@
 <a href="./page.html"></a>
 <a href=" ./page.html "></a>
 <a href=" /page.html "></a>
-<a href="//domain.com/something/"></a>
-<a href="http://domain.com/something/"></a>
-<a href="https://domain.com/something/"></a>
 <a href=""></a>
 <a href=":~:text=put%20your%20labels%20above%20your%20inputs">Sign-in form best practices</a>

packages/check-html-links/test-node/validateFolder.test.js

@@ -66,6 +66,44 @@ describe('validateFolder', () => {
     ]);
   });
 
+  it('validates external links', async () => {
+    const { errors, cleanup } = await execute('fixtures/external-link', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/external-link/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            attribute: 'href',
+            value: '//rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 6,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'http://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 7,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'https://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 8,
+            character: 9,
+            anchor: '',
+          },
+        ],
+      },
+    ]);
+  });
+
   it('groups multiple usage of the same missing file', async () => {
     const { errors, cleanup } = await execute('fixtures/internal-links-to-same-file');
     expect(cleanup(errors)).to.deep.equal([