add quoting=csv.QUOTE_NONE

Author: ChengYehLi

libmultilabel/linear/preprocessor.py

@@ -12,6 +12,8 @@
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.preprocessing import MultiLabelBinarizer
 
+import csv
+
 __all__ = ['Preprocessor']
 
 
@@ -141,7 +143,7 @@ def _generate_label_mapping(self, labels, classes=None):
 def read_libmultilabel_format(path: str) -> 'dict[str,list[str]]':
     data = pd.read_csv(path, sep='\t', header=None,
                        dtype=str,
-                       on_bad_lines='skip').fillna('')
+                       on_bad_lines='skip', quoting=csv.QUOTE_NONE).fillna('')
     if data.shape[1] == 2:
         data.columns = ['label', 'text']
         data = data.reset_index()

libmultilabel/nn/data_utils.py

@@ -13,6 +13,8 @@
 from torchtext.vocab import build_vocab_from_iterator, pretrained_aliases
 from tqdm import tqdm
 
+import csv
+
 transformers.logging.set_verbosity_error()
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
@@ -136,7 +138,7 @@ def _load_raw_data(path, is_test=False, tokenize_text=True, remove_no_label_data
     """
     logging.info(f'Load data from {path}.')
     data = pd.read_csv(path, sep='\t', header=None,
-                       error_bad_lines=False, warn_bad_lines=True).fillna('')
+                       error_bad_lines=False, warn_bad_lines=True, quoting=csv.QUOTE_NONE).fillna('')
     if data.shape[1] == 2:
         data.columns = ['label', 'text']
         data = data.reset_index()