Adding quail (#1208)

* adding quail

* removing extra files

* updating supported task markdown page

* adjusting supported task alphabetical ordering

Author: yzpang

guides/tasks/supported_tasks.md

@@ -11,6 +11,8 @@
 | SuperGLUE Winogender Diagnostic | superglue_axg | ✅ | ✅ | superglue_axg | SuperGLUE |
 | Acceptability Definiteness | acceptability_definiteness | ✅ |  | acceptability_definiteness | Function Words |
 | Adversarial NLI | `adversarial_nli_{round}` | ✅ |  | adversarial_nli | 3 rounds |
+| ARC ("easy" version) | arc_easy | ✅ | ✅ | arc_easy | [site](https://allenai.org/data/arc) |
+| ARC ("challenge" version) | arc_challenge | ✅ | ✅ | arc_challenge | [site](https://allenai.org/data/arc) |
 | BoolQ | boolq | ✅ | ✅ | boolq | SuperGLUE |
 | BUCC2018 | `bucc2018_{lang}` | ✅ | ✅ | bucc2018 | XTREME, multi-lang |
 | CommitmentBank | cb | ✅ | ✅ | cb | SuperGLUE |
@@ -25,18 +27,21 @@
 | EP-DPR | dpr | ✅ |  | dpr | Edge-Probing |
 | GLUE Diagnostic | glue_diagnostics | ✅ | ✅ | glue_diagnostics | GLUE |
 | HellaSwag | hellaswag | ✅ | ✅ | hellaswag |  |
-| [MCScript2.0](https://arxiv.org/pdf/1905.09531.pdf) | mcscript | ✅ |  | mcscript | [Data](https://my.hidrive.com/share/wdnind8pp5#$/) |
+| [MCScript2.0](https://arxiv.org/pdf/1905.09531.pdf) | mcscript | ✅ |  | mcscript | [data](https://my.hidrive.com/share/wdnind8pp5#$/) |
 | MLM | * | ✅ | * | mlm_simple | See task-specific notes. |
 | MLQA | `mlqa_{lang1}_{lang2}` | ✅ | ✅ | mlqa | XTREME, multi-lang |
 | MNLI | mnli | ✅ | ✅ | mnli | GLUE, MNLI-matched |
 | MNLI-mismatched | mnli_mismatched | ✅ | ✅ | mnli_mismatched | GLUE |
-| MultiRC | multirc | ✅ | ✅ | multirc | SuperGLUE |
 | MRPC | mrpc | ✅ | ✅ | mrpc | GLUE |
+| MultiRC | multirc | ✅ | ✅ | multirc | SuperGLUE |
+| Mutual (standard version) | mutual | ✅ |  | mutual | [site](https://github.com/Nealcly/MuTual) |
+| Mutual ("challenge" version) | mutual_plus | ✅ |  | mutual_plus | [site](https://github.com/Nealcly/MuTual) |
 | Natural Questions | mrqa_natural_questions | ✅ | ✅ | mrqa_natural_questions | [MRQA](https://mrqa.github.io/) version of task |
 | NewsQA | newsqa | ✅ | ✅ | newsqa |  |
 | PIQA | piqa | ✅ | ✅ | piqa | [PIQA](https://yonatanbisk.com/piqa/) |
 | QAMR | qamr | ✅ | ✅ | qamr |  |
 | QA-SRL | qasrl | ✅ | ✅ | qasrl |  |
+| Quail | quail | ✅ | | quail | [site](http://text-machine.cs.uml.edu/lab2/projects/quail/) |
 | Quoref | quoref | ✅ | ✅ | quoref |  |
 | EP-NER | ner | ✅ |  | ner | Edge-Probing |
 | PAWS-X | `pawsx_{lang}` | ✅ | ✅ | pawsx | XTREME, multi-lang |

jiant/tasks/evaluate/core.py

@@ -996,6 +996,7 @@ def get_evaluation_scheme_for_task(task) -> BaseEvaluationScheme:
             tasks.HellaSwagTask,
             tasks.MutualTask,
             tasks.MutualPlusTask,
+            tasks.QuailTask,
             tasks.SocialIQATask,
             tasks.MCTestTask,
         ),

jiant/tasks/lib/quail.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+
+from jiant.tasks.lib.templates.shared import labels_to_bimap
+from jiant.tasks.lib.templates import multiple_choice as mc_template
+from jiant.utils.python.io import read_json_lines
+
+
+@dataclass
+class Example(mc_template.Example):
+    @property
+    def task(self):
+        return QuailTask
+
+
+@dataclass
+class TokenizedExample(mc_template.TokenizedExample):
+    pass
+
+
+@dataclass
+class DataRow(mc_template.DataRow):
+    pass
+
+
+@dataclass
+class Batch(mc_template.Batch):
+    pass
+
+
+class QuailTask(mc_template.AbstractMultipleChoiceTask):
+    Example = Example
+    TokenizedExample = Example
+    DataRow = DataRow
+    Batch = Batch
+
+    CHOICE_KEYS = ["0", "1", "2", "3"]
+    CHOICE_TO_ID, ID_TO_CHOICE = labels_to_bimap(CHOICE_KEYS)
+    NUM_CHOICES = len(CHOICE_KEYS)
+
+    def get_train_examples(self):
+        return self._create_examples(lines=read_json_lines(self.train_path), set_type="train")
+
+    def get_val_examples(self):
+        return self._create_examples(lines=read_json_lines(self.val_path), set_type="val")
+
+    def get_test_examples(self):
+        return self._create_examples(lines=read_json_lines(self.test_path), set_type="test")
+
+    @classmethod
+    def _create_examples(cls, lines, set_type):
+        examples = []
+        for i, line in enumerate(lines):
+            examples.append(
+                Example(
+                    guid="%s-%s" % (set_type, i),
+                    prompt=line["context"],
+                    choice_list=[d for d in line["answers"]],
+                    label=line["label"],
+                )
+            )
+        return examples

jiant/tasks/retrieval.py

@@ -42,6 +42,7 @@
 from jiant.tasks.lib.qasrl import QASRLTask
 from jiant.tasks.lib.qqp import QqpTask
 from jiant.tasks.lib.qnli import QnliTask
+from jiant.tasks.lib.quail import QuailTask
 from jiant.tasks.lib.quoref import QuorefTask
 from jiant.tasks.lib.record import ReCoRDTask
 from jiant.tasks.lib.rte import RteTask
@@ -118,6 +119,7 @@
     "qasrl": QASRLTask,
     "qnli": QnliTask,
     "qqp": QqpTask,
+    "quail": QuailTask,
     "quoref": QuorefTask,
     "record": ReCoRDTask,
     "rte": RteTask,