Fix Encoding on Body (#143)

* fix: check encoding on body

Author: kev5873

dist/generate-custom-parser.js

@@ -43,7 +43,9 @@ if (process.env.CI) {
           assert.equal(article.title, result.title);
           done();
         }).catch((e) => {
-          console.log('THIS WENT WRONG', e); // eslint-disable-line no-console
+          console.log('There was an error', e.message); // eslint-disable-line no-console
+          console.log('e.fileName', e.fileName);
+          console.log('e.lineNumber', e.lineNumber);
           assert.equal(true, false);
           done();
         });

dist/generate-custom-parser.js.map

@@ -1,5 +1,7 @@
 import cheerio from 'cheerio';
+import iconv from 'iconv-lite';
 
+import { getEncoding } from 'utils/text';
 import { fetchResource } from './utils';
 import {
   normalizeMetaTags,
@@ -51,7 +53,7 @@ const Resource = {
       throw new Error('Content does not appear to be text.');
     }
 
-    let $ = cheerio.load(content);
+    let $ = this.encodeDoc({ content, contentType });
 
     if ($.root().children().length === 0) {
       throw new Error('No children, likely a bad parse.');
@@ -63,6 +65,24 @@ const Resource = {
 
     return $;
   },
+
+  encodeDoc({ content, contentType }) {
+    const encoding = getEncoding(contentType);
+    let decodedContent = iconv.decode(content, encoding);
+    let $ = cheerio.load(decodedContent);
+
+    // after first cheerio.load, check to see if encoding matches
+    const metaContentType = $('meta[http-equiv=content-type]').attr('content');
+    const properEncoding = getEncoding(metaContentType);
+
+    // if encodings in the header/body dont match, use the one in the body
+    if (properEncoding !== encoding) {
+      decodedContent = iconv.decode(content, properEncoding);
+      $ = cheerio.load(decodedContent);
+    }
+
+    return $;
+  },
 };
 
 export default Resource;

dist/mercury.js

@@ -1,6 +1,7 @@
 import assert from 'assert';
 import cheerio from 'cheerio';
 import { Errors } from 'utils';
+import { getEncoding } from 'utils/text';
 
 import { record } from 'test-helpers';
 import Resource from './index';
@@ -24,18 +25,31 @@ describe('Resource', () => {
 
       assert.equal(error, Errors.badUrl);
     });
-  });
 
-  describe('generateDoc({ body, response })', () => {
-    it('returns a cheerio object if valid', () => {
-      const response = { headers: { 'content-type': 'text/html' } };
+    it('fetches with different encoding on body', async () => {
+      const url = 'http://www.playnation.de/spiele-news/kojima-productions/hideo-kojima-reflektiert-ueber-seinen-werdegang-bei-konami-id68950.html';
+      const $ = await Resource.create(url);
+      const metaContentType = $('meta[http-equiv=content-type]').attr('value');
+
+      assert.equal(getEncoding(metaContentType), 'iso-8859-1');
+      const encodedU = /ü/g;
+
+      assert.equal(encodedU.test($.html()), true);
+      assert.equal(typeof $, 'function');
+    });
+
+    it('handles special encoding', async () => {
+      const url = 'http://www.elmundo.es/opinion/2016/11/19/582f476846163fc65a8b4578.html';
+      const $ = await Resource.create(url);
 
-      const body = '<div><p>Hi</p></div>';
-      const $ = Resource.generateDoc({ body, response });
+      const badEncodingRe = /�/g;
 
-      assert.equal($.html(), body);
+      assert.equal(badEncodingRe.test($.html()), false);
+      assert.equal(typeof $, 'function');
     });
+  });
 
+  describe('generateDoc({ body, response })', () => {
     it('throws an error if the content is not text', () => {
       const response = {
         headers: {

dist/mercury.js.map

@@ -1,8 +1,6 @@
 import URL from 'url';
 import request from 'request';
-import iconv from 'iconv-lite';
 import { Errors } from 'utils';
-import { getEncoding } from 'utils/text';
 
 import {
   REQUEST_HEADERS,
@@ -17,12 +15,6 @@ function get(options) {
       if (err) {
         reject(err);
       } else {
-        const encoding = getEncoding(response.headers['content-type']);
-
-        if (iconv.encodingExists(encoding)) {
-          body = iconv.decode(body, encoding);
-        }
-
         resolve({ body, response });
       }
     });
@@ -97,9 +89,6 @@ export default async function fetchResource(url, parsedUrl) {
     url: parsedUrl.href,
     headers: { ...REQUEST_HEADERS },
     timeout: FETCH_TIMEOUT,
-    // Don't set encoding; fixes issues
-    // w/gzipped responses
-    encoding: null,
     // Accept cookies
     jar: true,
     // Accept and decode gzip