Enable Fast tokenizer and add T5 reader

README.md

@@ -45,7 +45,7 @@ Below is a example for English Question-Answering. We also provide an example fo
 ```python
 from bertserini.reader.base import Question, Context
 from bertserini.reader.bert_reader import BERT
-from bertserini.utils.utils_new import get_best_answer
+from bertserini.utils.utils import get_best_answer
 
 model_name = "rsvp-ai/bertserini-bert-base-squad"
 tokenizer_name = "rsvp-ai/bertserini-bert-base-squad"

bertserini/experiments/eval/evaluate_v1.py

@@ -4,6 +4,12 @@
 import argparse
 import json
 
+from rouge_metric import PyRouge
+rouge = PyRouge(rouge_n=(2,), rouge_su=True, skip_gap=4)
+#from rouge_score import rouge_scorer
+#rouge1_scorer = rouge_scorer.RougeScorer(['rouge1'], use_stemmer=True)
+#rougel_scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
+
 from bertserini.utils.utils import normalize_answer, init_logger
 
 logger = init_logger("evluation")
@@ -67,6 +73,32 @@ def overlap_score(prediction, ground_truth):
 def exact_match_score(prediction, ground_truth):
     return normalize_answer(prediction) == normalize_answer(ground_truth)
 
+def rouge2_r_score(prediction, ground_truth):
+    if len(prediction) == 0:
+        return 0
+    return rouge.evaluate([ground_truth], [[prediction]])["rouge-2"]["r"]
+    #return rouge1_scorer.score(prediction, ground_truth)
+
+def rouge2_f_score(prediction, ground_truth):
+    if len(prediction) == 0:
+        return 0
+    return rouge.evaluate([ground_truth], [[prediction]])["rouge-2"]["f"]
+
+def rougesu4_r_score(prediction, ground_truth):
+    if len(prediction) == 0:
+        return 0
+    return rouge.evaluate([ground_truth], [[prediction]])["rouge-su4"]["r"]
+
+def rougesu4_f_score(prediction, ground_truth):
+    if len(prediction) == 0:
+        return 0
+    return rouge.evaluate([ground_truth], [[prediction]])["rouge-su4"]["f"]
+
+#def rougel_score(prediction, ground_truth):
+#    print(rougel_scorer.score(prediction, ground_truth))
+#    input()
+#    return rougel_scorer.score(prediction, ground_truth)
+
 
 def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
     scores_for_ground_truths = []
@@ -92,7 +124,7 @@ def metric_max_recall(metric_fn, prediction, ground_truths):
 
 
 def evaluate(dataset, predictions):
-    sentence_cover = precision = cover = sentence_recall = recall = f1 = exact_match = total = overlap = 0
+    sentence_cover = precision = cover = sentence_recall = recall = f1 = exact_match = total = overlap = rouge2_r = rouge2_f = rougesu4_r = rougesu4_f = 0
     for article in dataset:
         for paragraph in article['paragraphs']:
             for qa in paragraph['qas']:
@@ -104,6 +136,11 @@ def evaluate(dataset, predictions):
                 ground_truths = list(map(lambda x: x['text'], qa['answers']))
                 prediction = [predictions[qa['id']]]
                 #prediction_sentence = predictions[qa['id']]['sentences']
+                rouge2_r += metric_max_recall(rouge2_r_score,  prediction, ground_truths)
+                rouge2_f += metric_max_recall(rouge2_f_score,  prediction, ground_truths)
+                rougesu4_r += metric_max_recall(rougesu4_r_score,  prediction, ground_truths)
+                rougesu4_f += metric_max_recall(rougesu4_f_score,  prediction, ground_truths)
+                #rougel += metric_max_recall(rougel_score,  prediction, ground_truths)
                 cover += metric_max_recall(cover_score, prediction, ground_truths)
                 exact_match += metric_max_recall(
                     exact_match_score, prediction, ground_truths)
@@ -124,21 +161,27 @@ def evaluate(dataset, predictions):
     overlap = 100.0 * overlap / total
     cover = 100.0 * cover / total
     precision = 100.0 * precision / total
+    rouge2_r = 100.0 * rouge2_r / total
+    rouge2_f = 100.0 * rouge2_f / total
+    rougesu4_r = 100.0 * rougesu4_r / total
+    rougesu4_f = 100.0 * rougesu4_f / total
+    #rougel = 100.0 * rougel / total
     #sentence_recall = 100.0 * sentence_recall / total
     #sentence_cover = 100.0 * sentence_cover / total
 
     return {'exact_match': exact_match, 'f1': f1, "recall": recall, 
             #"sentence_recall": sentence_recall, "sentence_cover": sentence_cover,
-            "precision": precision, "cover": cover, "overlap": overlap}
+            "precision": precision, "cover": cover, "overlap": overlap, 
+            "rouge2_r": rouge2_r, "rouge2_f":rouge2_f, "rougesu4_r": rougesu4_r, "rougesu4_f": rougesu4_f}
 
 
 def squad_v1_eval(dataset_filename, prediction_filename):
     expected_version = '1.1'
     with open(dataset_filename) as dataset_file:
         dataset_json = json.load(dataset_file)
-        if dataset_json['version'] != expected_version:
-            logger.error('Evaluation expects v-{}, but got dataset with v-{}'.format(
-                expected_version, dataset_json['version']))
+        #if dataset_json['version'] != expected_version:
+        #    logger.error('Evaluation expects v-{}, but got dataset with v-{}'.format(
+        #        expected_version, dataset_json['version']))
         dataset = dataset_json['data']
     with open(prediction_filename) as prediction_file:
         predictions = json.load(prediction_file)

bertserini/experiments/inference.py

@@ -2,8 +2,9 @@
 from tqdm import tqdm
 from bertserini.reader.bert_reader import BERT
 from bertserini.retriever.pyserini_retriever import retriever, build_searcher
-from bertserini.utils.utils_new import extract_squad_questions
+from bertserini.utils.utils import extract_squad_questions
 from bertserini.experiments.args import *
+import time
 
 if __name__ == "__main__":
     questions = extract_squad_questions(args.dataset_path, do_strip_accents=args.strip_accents)
@@ -13,8 +14,11 @@
 
     all_answer = []
     for question in tqdm(questions):
+        # print("before retriever:", time.time())
         contexts = retriever(question, searcher, args.topk)
+        # print("before reader:", time.time())
         final_answers = bert_reader.predict(question, contexts)
+        # print("after reader:", time.time())
         final_answers_lst = []
         for ans in final_answers:
             final_answers_lst.append(