Fixes an issue with encoding (#154)

* fix: fixes an issue with encoding on the fetch level

Author: kev5873

dist/mercury.js

@@ -43,9 +43,7 @@ if (process.env.CI) {
           assert.equal(article.title, result.title);
           done();
         }).catch((e) => {
-          console.log('There was an error', e.message); // eslint-disable-line no-console
-          console.log('e.fileName', e.fileName);
-          console.log('e.lineNumber', e.lineNumber);
+          console.log(e.name, e.message); // eslint-disable-line no-console
           assert.equal(true, false);
           done();
         });

dist/mercury.js.map

@@ -50,6 +50,9 @@ describe('Resource', () => {
   });
 
   describe('generateDoc({ body, response })', () => {
+    // Ideally the body would be a buffer, because of potential issues with
+    // string re-encoding, since these strings are blank, it should be fine
+    // but this is why iconv is throwing warnings.
     it('throws an error if the content is not text', () => {
       const response = {
         headers: {

scripts/check-build.test.js

@@ -91,6 +91,9 @@ export default async function fetchResource(url, parsedUrl) {
     timeout: FETCH_TIMEOUT,
     // Accept cookies
     jar: true,
+    // Set to null so the response returns as binary and body as buffer
+    // https://github.com/request/request#requestoptions-callback
+    encoding: null,
     // Accept and decode gzip
     gzip: true,
     // Follow any redirect

src/resource/index.test.js

@@ -21,6 +21,13 @@ describe('fetchResource(url)', () => {
     assert.equal(error, true);
   });
 
+  it('returns a buffer as its body', async () => {
+    const url = 'https://www.washingtonpost.com/news/post-nation/wp/2016/11/05/a-vile-and-disgusting-act-officer-accused-of-giving-fecal-sandwich-to-homeless-man-is-fired/';
+    const result = await fetchResource(url);
+
+    assert.equal(typeof result.body, 'object');
+  });
+
   it('fetches nyt', async () => {
     const url = 'http://www.nytimes.com/2016/08/16/upshot/the-state-of-the-clinton-trump-race-is-it-over.html?_r=0';
     const { response } = await fetchResource(url);