Fix eval ref miss in Chinese WWM.

examples/language-modeling/README.md

@@ -63,7 +63,7 @@ python run_language_modeling.py \
     --whole_word_mask
 ```
 
-For Chinese models, it's same with English model with only --mlm`. If using whole-word masking, we need to generate a reference files, case it's char level.
+For Chinese models, it's same with English model with only `--mlm`. If using whole-word masking, we need to generate a reference files, cause it's char level.
 
 **Q :** Why ref file ?
 
@@ -76,6 +76,9 @@ So we need a ref file to tell model which pos of BERT original token should be a
 **A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm) by HIT. It works well on so many Chines Task like CLUE (Chinese GLUE).
 They use LTP, so if we want to fine-tune their model, we need LTP.
 
+Now LTP only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
+
+You have to run `chinese_ref.py` on older version, but `run_language_modeling.py` on newest version. 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
 export LTP_RESOURCE=/path/to/ltp/tokenizer
@@ -84,7 +87,7 @@ export SAVE_PATH=/path/to/data/ref.txt
 
 python chinese_ref.py \
     --file_name=$TRAIN_FILE \
-    --ltp=$LTP_RESOURCE
+    --ltp=$LTP_RESOURCE \
     --bert=$BERT_RESOURCE \
     --save_path=$SAVE_PATH 
 ```

examples/language-modeling/run_language_modeling.py

@@ -103,9 +103,11 @@ class DataTrainingArguments:
         default=None,
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
-    chinese_ref_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input ref data file for whole word mask in Chinees."},
+    train_ref_file: Optional[str] = field(
+        default=None, metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
+    )
+    eval_ref_file: Optional[str] = field(
+        default=None, metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
     )
     line_by_line: bool = field(
         default=False,
@@ -148,16 +150,13 @@ def get_dataset(
     evaluate: bool = False,
     cache_dir: Optional[str] = None,
 ):
-    def _dataset(file_path):
+    def _dataset(file_path, ref_path=None):
         if args.line_by_line:
-            if args.chinese_ref_file is not None:
+            if ref_path is not None:
                 if not args.whole_word_mask or not args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
-                    tokenizer=tokenizer,
-                    file_path=file_path,
-                    block_size=args.block_size,
-                    ref_path=args.chinese_ref_file,
+                    tokenizer=tokenizer, file_path=file_path, block_size=args.block_size, ref_path=ref_path,
                 )
 
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
@@ -171,11 +170,11 @@ def _dataset(file_path):
             )
 
     if evaluate:
-        return _dataset(args.eval_data_file)
+        return _dataset(args.eval_data_file, args.eval_ref_file)
     elif args.train_data_files:
         return ConcatDataset([_dataset(f) for f in glob(args.train_data_files)])
     else:
-        return _dataset(args.train_data_file)
+        return _dataset(args.train_data_file, args.train_ref_file)
 
 
 def main():
@@ -281,9 +280,7 @@ def main():
     )
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer,
-            plm_probability=data_args.plm_probability,
-            max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
         if data_args.mlm and data_args.whole_word_mask: