fix: kept elements being removed (#166)

Elements marked to keep were removeable under specific circumstances.
This PR fixes these edge cases.

Author: adampash

fixtures/obamawhitehouse.archives.gov/1490227791307.html

@@ -31,6 +31,9 @@ export default function extractCleanNode(
   // this can sometimes be too aggressive.
   if (defaultCleaner) cleanImages(article, $);
 
+  // Make links absolute
+  makeLinksAbsolute(article, $, url);
+
   // Mark elements to keep that would normally be removed.
   // E.g., stripJunkTags will remove iframes, so we're going to mark
   // YouTube/Vimeo videos as elements we want to keep.
@@ -48,9 +51,6 @@ export default function extractCleanNode(
   // Clean headers
   cleanHeaders(article, $, title);
 
-  // Make links absolute
-  makeLinksAbsolute(article, $, url);
-
   // We used to clean UL's and OL's here, but it was leading to
   // too many in-article lists being removed. Consider a better
   // way to detect menus particularly and remove them.

src/cleaners/content.js

@@ -46,9 +46,6 @@ export const ObamawhitehouseArchivesGovExtractor = {
     // Is there anything in the content you selected that needs transformed
     // before it's consumable content? E.g., unusual lazy loaded images
     transforms: {
-      'iframe[src*=youtube]': ($node) => {
-        $node.parents('.panel-pane').replaceWith($node);
-      },
     },
 
     // Is there anything that is in the result that shouldn't be?

src/extractors/custom/obamawhitehouse.archives.gov/index.js

@@ -115,4 +115,30 @@ describe('ObamawhitehouseArchivesGovExtractor', () => {
       assert.equal($('iframe[src*="youtube"]').length, 1);
     });
   });
+
+  describe('gets more failing blogs', () => {
+    let result;
+    let url;
+    beforeAll(() => {
+      url =
+        'https://obamawhitehouse.archives.gov/the-press-office/2016/12/24/weekly-address-merry-christmas-president-and-first-lady';
+      const html =
+        fs.readFileSync('./fixtures/obamawhitehouse.archives.gov/1490227791307.html');
+      result =
+        Mercury.parse(url, html, { fallback: false });
+    });
+
+    it('gets the words and video', async () => {
+      const { content } = await result;
+
+      const $ = cheerio.load(content || '');
+
+      const first13 = excerptContent($('*').first().text(), 13);
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(first13, 'In this week’s address, the President and the First Lady wished all Americans');
+      assert.equal($('iframe[src*="youtube"]').length, 1);
+    });
+  });
 });

src/extractors/custom/obamawhitehouse.archives.gov/index.test.js

@@ -3,9 +3,12 @@ import {
   setAttrs,
 } from 'utils/dom';
 
-import { WHITELIST_ATTRS_RE } from './constants';
+import {
+  WHITELIST_ATTRS_RE,
+  KEEP_CLASS,
+} from './constants';
 
-function removeAllButWhitelist($article) {
+function removeAllButWhitelist($article, $) {
   $article.find('*').each((index, node) => {
     const attrs = getAttrs(node);
 
@@ -18,6 +21,9 @@ function removeAllButWhitelist($article) {
     }, {}));
   });
 
+  // Remove the mercury-parser-keep class from result
+  $(`.${KEEP_CLASS}`, $article).removeClass(KEEP_CLASS);
+
   return $article;
 }
 
@@ -28,12 +34,12 @@ function removeAllButWhitelist($article) {
 // }
 
 // Remove attributes like style or align
-export default function cleanAttributes($article) {
+export default function cleanAttributes($article, $) {
   // Grabbing the parent because at this point
   // $article will be wrapped in a div which will
   // have a score set on it.
   return removeAllButWhitelist(
-    $article.parent().length ?
-      $article.parent() : $article
+    $article.parent().length ? $article.parent() : $article,
+    $,
   );
 }

src/utils/dom/clean-attributes.js

@@ -9,14 +9,14 @@ describe('cleanAttributes($)', () => {
   it('removes style attributes from nodes', () => {
     const $ = cheerio.load(HTML.removeStyle.before);
 
-    const result = cleanAttributes($('*').first());
+    const result = cleanAttributes($('*').first(), $);
     assertClean($.html(result), HTML.removeStyle.after);
   });
 
   it('removes align attributes from nodes', () => {
     const $ = cheerio.load(HTML.removeAlign.before);
 
-    const result = cleanAttributes($('*').first());
+    const result = cleanAttributes($('*').first(), $);
     assertClean($.html(result), HTML.removeAlign.after);
   });
 });

src/utils/dom/clean-attributes.test.js

@@ -5,7 +5,10 @@ import {
   scoreCommas,
 } from 'extractors/generic/content/scoring';
 
-import { CLEAN_CONDITIONALLY_TAGS } from './constants';
+import {
+  CLEAN_CONDITIONALLY_TAGS,
+  KEEP_CLASS,
+} from './constants';
 import { normalizeSpaces } from '../text';
 import { linkDensity } from './index';
 
@@ -89,6 +92,9 @@ function removeUnlessContent($node, $, weight) {
 export default function cleanTags($article, $) {
   $(CLEAN_CONDITIONALLY_TAGS, $article).each((index, node) => {
     const $node = $(node);
+    // If marked to keep, skip it
+    if ($node.hasClass(KEEP_CLASS) || $node.find(`.${KEEP_CLASS}`).length > 0) return;
+
     let weight = getScore($node);
     if (!weight) {
       weight = getOrInitScore($node, $);

src/utils/dom/clean-tags.js

@@ -12,8 +12,5 @@ export default function stripJunkTags(article, $, tags = []) {
   // any element with a class of mercury-parser-keep
   $(tags.join(','), article).not(`.${KEEP_CLASS}`).remove();
 
-  // Remove the mercury-parser-keep class from result
-  $(`.${KEEP_CLASS}`, article).removeClass(KEEP_CLASS);
-
   return $;
 }