handle use case in which no fields are quoted

Author: drewbanin

README.md

@@ -136,6 +136,10 @@ See below for an exhaustive list of configuration fields:
             // if true, unzip the file before reading it at a csv
             "unzip": true,
 
+            // if specified, override the default CSV quoting config
+            // More info: https://docs.python.org/3/library/csv.html#csv.QUOTE_ALL
+            "quoting": "QUOTE_NONE",
+
             // if the files don't have a header row, you can specify the field names
             "field_names": ["id", "first_name", "last_name"],
 

tap_s3_csv/config.py

@@ -15,6 +15,7 @@
         Required('format'): Any('csv', 'excel'),
         Optional('unzip'): bool,
         Optional('delimiter'): str,
+        Optional('quoting'): Any('QUOTE_MINIMAL', 'QUOTE_ALL', 'QUOTE_NONNUMERIC', 'QUOTE_NONE'),
         Optional('search_prefix'): str,
         Optional('field_names'): [str],
         Optional('worksheet_name'): str,

tap_s3_csv/csv_handler.py

@@ -1,9 +1,7 @@
 import codecs
 import csv
 import re
-
-import zlib
-import io
+import gzip
 
 
 def generator_wrapper(reader):
@@ -27,30 +25,9 @@ def generator_wrapper(reader):
         yield to_return
 
 
-def gunzip(stream):
-    dec = zlib.decompressobj(32 + zlib.MAX_WBITS)
-    for chunk in stream:
-        rv = dec.decompress(chunk)
-        if rv:
-            yield rv
-
-
-def iter_lines(stream):
-    buf = ""
-    for chunk in stream:
-        for byte in chunk:
-            char = chr(byte)
-            if char == '\n':
-                yield buf.encode('utf-8')
-                buf = ""
-            else:
-                buf += char
-
-
 def get_row_iterator(table_spec, file_handle):
-
     if table_spec.get('unzip'):
-        raw_stream = iter_lines(gunzip(file_handle._raw_stream))
+        raw_stream = gzip.GzipFile(fileobj=file_handle._raw_stream)
     else:
         raw_stream = file_handle._raw_stream
 
@@ -66,6 +43,10 @@ def get_row_iterator(table_spec, file_handle):
         field_names = table_spec['field_names']
 
     delimiter = table_spec.get('delimiter', ',')
-    reader = csv.DictReader(file_stream, delimiter=delimiter, fieldnames=field_names)
+
+    quote_config = table_spec.get('quote_config', 'QUOTE_MINIMAL')
+    quoting = getattr(csv, quote_config)
+
+    reader = csv.DictReader(file_stream, quoting=quoting, delimiter=delimiter, fieldnames=field_names)
 
     return generator_wrapper(reader)