From 4b317d61dc5dd0fbba2bedcff423d1947a12aeca Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 14 Mar 2022 00:21:58 -0700 Subject: [PATCH 01/91] Draft PR for bigbench datasets --- datasets/bigbench/README.md | 127 +++++++++++++++++++++ datasets/bigbench/bigbench.py | 207 ++++++++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+) create mode 100644 datasets/bigbench/README.md create mode 100644 datasets/bigbench/bigbench.py diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md new file mode 100644 index 00000000000..f4dbd1c7b95 --- /dev/null +++ b/datasets/bigbench/README.md @@ -0,0 +1,127 @@ +--- +YAML tags: +- copy-paste the tags obtained with the tagging app: https://github.com/huggingface/datasets-tagging +--- + +# Dataset Card for [Dataset Name] + +## Table of Contents +- [Table of Contents](#table-of-contents) +- [Dataset Description](#dataset-description) + - [Dataset Summary](#dataset-summary) + - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards) + - [Languages](#languages) +- [Dataset Structure](#dataset-structure) + - [Data Instances](#data-instances) + - [Data Fields](#data-fields) + - [Data Splits](#data-splits) +- [Dataset Creation](#dataset-creation) + - [Curation Rationale](#curation-rationale) + - [Source Data](#source-data) + - [Annotations](#annotations) + - [Personal and Sensitive Information](#personal-and-sensitive-information) +- [Considerations for Using the Data](#considerations-for-using-the-data) + - [Social Impact of Dataset](#social-impact-of-dataset) + - [Discussion of Biases](#discussion-of-biases) + - [Other Known Limitations](#other-known-limitations) +- [Additional Information](#additional-information) + - [Dataset Curators](#dataset-curators) + - [Licensing Information](#licensing-information) + - [Citation Information](#citation-information) + - [Contributions](#contributions) + +## Dataset Description + +- **Homepage:** +- **Repository:** +- **Paper:** +- **Leaderboard:** +- **Point of Contact:** + +### Dataset Summary + +[More Information Needed] + +### Supported Tasks and Leaderboards + +[More Information Needed] + +### Languages + +[More Information Needed] + +## Dataset Structure + +### Data Instances + +[More Information Needed] + +### Data Fields + +[More Information Needed] + +### Data Splits + +[More Information Needed] + +## Dataset Creation + +### Curation Rationale + +[More Information Needed] + +### Source Data + +#### Initial Data Collection and Normalization + +[More Information Needed] + +#### Who are the source language producers? + +[More Information Needed] + +### Annotations + +#### Annotation process + +[More Information Needed] + +#### Who are the annotators? + +[More Information Needed] + +### Personal and Sensitive Information + +[More Information Needed] + +## Considerations for Using the Data + +### Social Impact of Dataset + +[More Information Needed] + +### Discussion of Biases + +[More Information Needed] + +### Other Known Limitations + +[More Information Needed] + +## Additional Information + +### Dataset Curators + +[More Information Needed] + +### Licensing Information + +[More Information Needed] + +### Citation Information + +[More Information Needed] + +### Contributions + +Thanks to [@github-username](https://github.com/) for adding this dataset. diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py new file mode 100644 index 00000000000..390f2ccd4dd --- /dev/null +++ b/datasets/bigbench/bigbench.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""HuggingFace datasets implementation of the json tasks in the BIG-Bench Dataset. +For the programatic tasks, please use the BIG-Bench API on github.com/google/BIG-bench. +""" + + +import csv +import json +import numpy as np +import os +from typing import Optional + +import datasets + +try: + import bigbench.bbseqio.bigbench_bridge as bbb + from bigbench.bbseqio import bigbench_json_paths as bb_json_paths +except ModuleNotFoundError as error: + print(f"Failed to import bigbench. Please see https://github.com/google/BIG-bench or install using 'pip install git+https://github.com/google/BIG-bench.git'") + raise + +_CITATION = """\ +@InProceedings{bigbench, +title = {Beyond the Imitation Game: Quantifying and extrapolating the +capabilities of language models}, +author={BIG-Bench Collaboration +}, +year={2022} +} +""" + +_DESCRIPTION = """\ +The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to +probe large language models, and extrapolate their future capabilities. +""" + +_HOMEPAGE = "https://github.com/google/BIG-bench" + +_LICENSE = "Apache License 2.0" + +def div_or_none(x, y): + return x // y if x else x + +_GLOBAL_MIN_VALIDATION_EXAMPLES = 16 + +class BigBenchConfig(datasets.BuilderConfig): + def __init__(self, + task_name: str, + + *args, + subtask_name: Optional[str] = None, + num_shots: int = 0, + max_examples: Optional[int] = None, + **kwargs): + super().__init__( + *args, + **kwargs, + ) + self.task_name = task_name + self.subtask_name = subtask_name + self.num_shots = num_shots + self.max_examples = max_examples + + +class BigBench(datasets.GeneratorBasedBuilder): + """The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark + intended to probe large language models, and extrapolate their future capabilities.""" + + VERSION = datasets.Version("1.0.0") + + BUILDER_CONFIG_CLASS = BigBenchConfig + + + def _info(self): + features = datasets.Features( + { + "idx": datasets.Value("int32"), + "inputs": datasets.Value("string"), + "targets": datasets.Sequence(datasets.Value("string")), + "multiple_choice_targets": datasets.Sequence(datasets.Value("string")), + "multiple_choice_scores": datasets.Sequence(datasets.Value("int32")), + }) + return datasets.DatasetInfo( + # This is the description that will appear on the datasets page. + description=_DESCRIPTION, + # This defines the different columns of the dataset and their types + features=features, # Here we define them above because they are different between the two configurations + # If there's a common (input, target) tuple from the features, + # specify them here. They'll be used if as_supervised=True in + # builder.as_dataset. + supervised_keys=None, + # Homepage of the dataset for documentation + homepage=_HOMEPAGE, + # License for the dataset if available + license=_LICENSE, + # Citation for the dataset + citation=_CITATION, + ) + + + def _split_generators(self, dl_manager): + """Returns SplitGenerators.""" + + return [ + # TODO(ajandreassen): Is there a way to implement all in HF? + # datasets.SplitGenerator( + # name=datasets.Split('default'), + # # These kwargs will be passed to _generate_examples + # gen_kwargs={ + # "split": "all", + # }, + # ), + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "split": "validation", + }, + ), + ] + + def _generate_examples( + self, split # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + ): + """ Yields examples as (key, example) tuples. """ + task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) + + has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) + + if not has_subtasks: + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + task_path = task_path, + subtask_name = None, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, + min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES + ) + ds = ds_fn(split) + elif self.config.subtask_name is not None: + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + task_path = task_path, + subtask_name = self.config.subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, + min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES + ) + ds = ds_fn(split) + else: + # Create mixture of all subtasks + subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) + num_subtasks = len(subtask_names) + max_examples_per_subtask = div_or_none(self.config.max_examples, num_subtasks) + min_validation_examples_per_subtask = div_or_none( + _GLOBAL_MIN_VALIDATION_EXAMPLES, num_subtasks) + subtask_datasets = [] + for subtask_name in subtask_names: + subtask_name = subtask_name.split(':')[-1] + print(f"Loading subtask {split} split", subtask_name) + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + task_path = task_path, + subtask_name = subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = max_examples_per_subtask, + json_util = json_util, + min_validation_examples = min_validation_examples_per_subtask + ) + subtask_datasets.append(ds_fn(split)) + ds = subtask_datasets[0] + for sub_ds in subtask_datasets[1:]: + ds = ds.concatenate(sub_ds) + + unique_key_counter = 0 + + for example in ds: + unique_key_counter +=1 + yield unique_key_counter, { + "idx": example["idx"], + "inputs": example["inputs"], + "targets": example["targets"], + "multiple_choice_targets": example["multiple_choice_targets"], + "multiple_choice_scores": example["multiple_choice_scores"], + } From 241a4c2d661c8f897c5738392bbeca652409f6f5 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 14 Mar 2022 11:33:03 -0700 Subject: [PATCH 02/91] Update handeling tasks with non-rectangular targets --- datasets/bigbench/bigbench.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 390f2ccd4dd..58c16137f9e 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -146,7 +146,7 @@ def _generate_examples( task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) - + if not has_subtasks: ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, task_path = task_path, @@ -157,7 +157,7 @@ def _generate_examples( json_util = json_util, min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES ) - ds = ds_fn(split) + ds_list = [ds_fn(split)] elif self.config.subtask_name is not None: ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, task_path = task_path, @@ -168,7 +168,7 @@ def _generate_examples( json_util = json_util, min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES ) - ds = ds_fn(split) + ds_list = [ds_fn(split)] else: # Create mixture of all subtasks subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) @@ -176,7 +176,7 @@ def _generate_examples( max_examples_per_subtask = div_or_none(self.config.max_examples, num_subtasks) min_validation_examples_per_subtask = div_or_none( _GLOBAL_MIN_VALIDATION_EXAMPLES, num_subtasks) - subtask_datasets = [] + ds_list = [] for subtask_name in subtask_names: subtask_name = subtask_name.split(':')[-1] print(f"Loading subtask {split} split", subtask_name) @@ -189,19 +189,16 @@ def _generate_examples( json_util = json_util, min_validation_examples = min_validation_examples_per_subtask ) - subtask_datasets.append(ds_fn(split)) - ds = subtask_datasets[0] - for sub_ds in subtask_datasets[1:]: - ds = ds.concatenate(sub_ds) + ds_list.append(ds_fn(split)) unique_key_counter = 0 - - for example in ds: - unique_key_counter +=1 - yield unique_key_counter, { - "idx": example["idx"], - "inputs": example["inputs"], - "targets": example["targets"], - "multiple_choice_targets": example["multiple_choice_targets"], - "multiple_choice_scores": example["multiple_choice_scores"], - } + for ds in ds_list: + for example in ds: + unique_key_counter +=1 + yield unique_key_counter, { + "idx": example["idx"], + "inputs": example["inputs"].numpy().decode(), + "targets": [target.numpy().decode() for target in example["targets"]], + "multiple_choice_targets": [targets.decode() for targets in example["multiple_choice_targets"].numpy()], + "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], + } From be1c6cafa77dce6671e92130aa77641b84feaf8b Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Wed, 6 Apr 2022 12:30:35 -0700 Subject: [PATCH 03/91] updates before PR --- datasets/bigbench/bigbench.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 58c16137f9e..c5744d18905 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -28,6 +28,8 @@ try: import bigbench.bbseqio.bigbench_bridge as bbb from bigbench.bbseqio import bigbench_json_paths as bb_json_paths + import bigbench.api.util as bb_utils + except ModuleNotFoundError as error: print(f"Failed to import bigbench. Please see https://github.com/google/BIG-bench or install using 'pip install git+https://github.com/google/BIG-bench.git'") raise @@ -56,6 +58,29 @@ def div_or_none(x, y): _GLOBAL_MIN_VALIDATION_EXAMPLES = 16 +def validate_task_name(task_name: str) -> None: + """Check that the requested task name is a valid bigbench json task.""" + if task_name in bb_utils.get_all_json_task_names(): + pass + elif task_name in bb_utils.get_all_programmatic_task_names(): + raise ValueError("BIG-Bench does not support programmatic tasks through HuggingFace datasets" + f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks.") + else: + print("Invalid task_name. Please choose one from:") + for name in bb_utils.get_all_json_task_names(): + print(f"-- {name}") + raise ValueError(f"Unknown task name. Got {task_name}.") + +def validate_subtask_name(task_name: str, subtask_name: str) -> None: + """Check that the requested subtask name is a valid bigbench subtask.""" + subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] + if subtask_name not in subtasks: + print("Invalid subtask_name for task {task_name}. Please choose one from:") + for name in subtasks: + print(f"-- {name}") + raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") + + class BigBenchConfig(datasets.BuilderConfig): def __init__(self, task_name: str, @@ -73,6 +98,10 @@ def __init__(self, self.subtask_name = subtask_name self.num_shots = num_shots self.max_examples = max_examples + + validate_task_name(self.task_name) + if self.subtask_name: + validate_subtask_name(self.task_name, self.subtask_name) class BigBench(datasets.GeneratorBasedBuilder): @@ -144,7 +173,7 @@ def _generate_examples( ): """ Yields examples as (key, example) tuples. """ task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) - + has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) if not has_subtasks: From 575e3839688e15501da101cf217fe9df5a3a0744 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 06:58:51 -0700 Subject: [PATCH 04/91] Modifying behavior for min num examples --- datasets/bigbench/bigbench.py | 127 +++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 56 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index c5744d18905..b3288df267c 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -53,10 +53,10 @@ _LICENSE = "Apache License 2.0" + def div_or_none(x, y): return x // y if x else x -_GLOBAL_MIN_VALIDATION_EXAMPLES = 16 def validate_task_name(task_name: str) -> None: """Check that the requested task name is a valid bigbench json task.""" @@ -75,7 +75,7 @@ def validate_subtask_name(task_name: str, subtask_name: str) -> None: """Check that the requested subtask name is a valid bigbench subtask.""" subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] if subtask_name not in subtasks: - print("Invalid subtask_name for task {task_name}. Please choose one from:") + print(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") for name in subtasks: print(f"-- {name}") raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") @@ -84,7 +84,6 @@ def validate_subtask_name(task_name: str, subtask_name: str) -> None: class BigBenchConfig(datasets.BuilderConfig): def __init__(self, task_name: str, - *args, subtask_name: Optional[str] = None, num_shots: int = 0, @@ -101,6 +100,9 @@ def __init__(self, validate_task_name(self.task_name) if self.subtask_name: + # Subtasks are sometimes in bigbench written as task_name:subtask_name. + # We want to remove the task_name from the subtask names: + self.subtask_name = self.subtask_name.split(':')[-1] validate_subtask_name(self.task_name, self.subtask_name) @@ -144,15 +146,14 @@ def _split_generators(self, dl_manager): """Returns SplitGenerators.""" return [ - # TODO(ajandreassen): Is there a way to implement all in HF? - # datasets.SplitGenerator( - # name=datasets.Split('default'), - # # These kwargs will be passed to _generate_examples - # gen_kwargs={ - # "split": "all", - # }, - # ), datasets.SplitGenerator( + name=datasets.splits.NamedSplit('default'), # TODO(ajandreassen): Is there a way override and call this 'all'? + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "split": "all", + }, + ), + datasets.SplitGenerator( name=datasets.Split.TRAIN, # These kwargs will be passed to _generate_examples gen_kwargs={ @@ -172,53 +173,67 @@ def _generate_examples( self, split # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` ): """ Yields examples as (key, example) tuples. """ - task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) - - has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) - - if not has_subtasks: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = None, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, - min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES - ) - ds_list = [ds_fn(split)] - elif self.config.subtask_name is not None: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = self.config.subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, - min_validation_examples = _GLOBAL_MIN_VALIDATION_EXAMPLES - ) - ds_list = [ds_fn(split)] + if split == 'all': + # not cutoff in number of examples for 'all' split + MIN_VALIDATION_EXAMPLES = 0 else: - # Create mixture of all subtasks - subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) - num_subtasks = len(subtask_names) - max_examples_per_subtask = div_or_none(self.config.max_examples, num_subtasks) - min_validation_examples_per_subtask = div_or_none( - _GLOBAL_MIN_VALIDATION_EXAMPLES, num_subtasks) - ds_list = [] - for subtask_name in subtask_names: - subtask_name = subtask_name.split(':')[-1] - print(f"Loading subtask {split} split", subtask_name) + MIN_VALIDATION_EXAMPLES = 16 + + try: + task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) + + has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) + if has_subtasks: + subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) + num_subtasks = len(subtask_names) + min_validation_examples_per_subtask = div_or_none( + MIN_VALIDATION_EXAMPLES, num_subtasks) + + if not has_subtasks: + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + task_path = task_path, + subtask_name = None, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, + min_validation_examples = MIN_VALIDATION_EXAMPLES + ) + ds_list = [ds_fn(split)] + elif self.config.subtask_name is not None: ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = max_examples_per_subtask, - json_util = json_util, - min_validation_examples = min_validation_examples_per_subtask - ) - ds_list.append(ds_fn(split)) + task_path = task_path, + subtask_name = self.config.subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, + min_validation_examples = min_validation_examples_per_subtask) + ds_list = [ds_fn(split)] + else: + # Create mixture of all subtasks + ds_list = [] + for subtask_name in subtask_names: + subtask_name = subtask_name.split(':')[-1] + print(f"Loading subtask {split} split", subtask_name) + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + task_path = task_path, + subtask_name = subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = div_or_none(self.config.max_examples, num_subtasks), + json_util = json_util, + min_validation_examples = min_validation_examples_per_subtask + ) + ds_list.append(ds_fn(split)) + except ValueError as value_error: + # BIG-Bench requires at least 16 examples to use the train & validation splits, + # while using 'all'/'default' does not have such a requirement. + if "has too few examples" in value_error.args[0] and split != 'all': + print(f'-- WARNING: skipping split {split} because it has too few examples.') + print(value_error) + return + raise value_error unique_key_counter = 0 for ds in ds_list: From 510709646243b3d1a5436d797451823f6cbaa1e0 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 07:13:11 -0700 Subject: [PATCH 05/91] Updating error message --- datasets/bigbench/bigbench.py | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index b3288df267c..a22b85b56e6 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -190,24 +190,24 @@ def _generate_examples( MIN_VALIDATION_EXAMPLES, num_subtasks) if not has_subtasks: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, task_path = task_path, - subtask_name = None, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, + subtask_name = None, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, min_validation_examples = MIN_VALIDATION_EXAMPLES ) ds_list = [ds_fn(split)] elif self.config.subtask_name is not None: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, task_path = task_path, - subtask_name = self.config.subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, + subtask_name = self.config.subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = self.config.max_examples, + json_util = json_util, min_validation_examples = min_validation_examples_per_subtask) ds_list = [ds_fn(split)] else: @@ -216,13 +216,13 @@ def _generate_examples( for subtask_name in subtask_names: subtask_name = subtask_name.split(':')[-1] print(f"Loading subtask {split} split", subtask_name) - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, + ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, task_path = task_path, - subtask_name = subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = div_or_none(self.config.max_examples, num_subtasks), - json_util = json_util, + subtask_name = subtask_name, + num_shots = self.config.num_shots, + bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, + max_examples = div_or_none(self.config.max_examples, num_subtasks), + json_util = json_util, min_validation_examples = min_validation_examples_per_subtask ) ds_list.append(ds_fn(split)) @@ -230,7 +230,7 @@ def _generate_examples( # BIG-Bench requires at least 16 examples to use the train & validation splits, # while using 'all'/'default' does not have such a requirement. if "has too few examples" in value_error.args[0] and split != 'all': - print(f'-- WARNING: skipping split {split} because it has too few examples.') + print(f"-- WARNING: skipping split {split} because it has too few examples. Please use 'default' split.") print(value_error) return raise value_error From 58774a40b58f955dbc6ed6e9f780fcb06b19a132 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 11:02:21 -0700 Subject: [PATCH 06/91] adding strip to inputs and targets to match bbseqio implementation --- datasets/bigbench/bigbench.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index a22b85b56e6..260d6479cc4 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -241,8 +241,8 @@ def _generate_examples( unique_key_counter +=1 yield unique_key_counter, { "idx": example["idx"], - "inputs": example["inputs"].numpy().decode(), - "targets": [target.numpy().decode() for target in example["targets"]], - "multiple_choice_targets": [targets.decode() for targets in example["multiple_choice_targets"].numpy()], + "inputs": example["inputs"].numpy().decode().strip(), + "targets": [target.numpy().decode().strip() for target in example["targets"]], + "multiple_choice_targets": [targets.decode().strip() for targets in example["multiple_choice_targets"].numpy()], "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], } From ca11c220acd2ca12a547e4ec03f88fcdd4e89a4e Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 11:02:46 -0700 Subject: [PATCH 07/91] Updated readme --- datasets/bigbench/README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index f4dbd1c7b95..2adb75357a3 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -3,7 +3,7 @@ YAML tags: - copy-paste the tags obtained with the tagging app: https://github.com/huggingface/datasets-tagging --- -# Dataset Card for [Dataset Name] +# Dataset Card for BIG-bench ## Table of Contents - [Table of Contents](#table-of-contents) @@ -27,24 +27,28 @@ YAML tags: - [Additional Information](#additional-information) - [Dataset Curators](#dataset-curators) - [Licensing Information](#licensing-information) - - [Citation Information](#citation-information) + - [Citation Inform ation](#citation-information) - [Contributions](#contributions) ## Dataset Description -- **Homepage:** -- **Repository:** -- **Paper:** +- **Homepage/Repository:** [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench) +- **Paper:** In progress - **Leaderboard:** -- **Point of Contact:** +- **Point of Contact:** [bigbench@googlegroups.com](mailto:bigbench@googlegroups.com) ### Dataset Summary -[More Information Needed] +The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models and extrapolate their future capabilities. Tasks included in BIG-bench are summarized by keyword [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md), and by task name [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/README.md). A paper introducing the benchmark, including evaluation results on large language models, is currently in preparation. ### Supported Tasks and Leaderboards -[More Information Needed] +BIG-Bench consists of both json and programmatic tasks. +This implementation in HuggingFace datasets implements + - 24 BIG-bench Lite tasks + - 167 BIG-bench json tasks (includes BIG-bench Lite) + +To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo](https://github.com/google/BIG-bench) ### Languages @@ -116,7 +120,7 @@ YAML tags: ### Licensing Information -[More Information Needed] +[Apache License 2.0](https://github.com/google/BIG-bench/blob/main/LICENSE) ### Citation Information @@ -124,4 +128,4 @@ YAML tags: ### Contributions -Thanks to [@github-username](https://github.com/) for adding this dataset. +Thanks to [@andersjohanandreassen](https://github.com/andersjohanandreassen) for adding this dataset. From d03005febcf59b45ab779ea2cebea32ca1ab2901 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 15:30:13 -0700 Subject: [PATCH 08/91] added config doc string --- datasets/bigbench/bigbench.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 260d6479cc4..86697b47344 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -93,6 +93,14 @@ def __init__(self, *args, **kwargs, ) + """BIG-bench configuration. + + Args: + task_name: BIG-bench task name. + subtask_name: BIG-bench subtask name. Accepts both "task_name:subtask_name" and "subtask_name" formats. + num_shots: Number of few-shot examples in input prompt. Default is zero. + max_examples: Limit number of examples for each task. Default is including all examples. + """ self.task_name = task_name self.subtask_name = subtask_name self.num_shots = num_shots From 497a9d8d3a85ed57e58f091f962db24ddb804584 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 16:18:28 -0700 Subject: [PATCH 09/91] trimming linewidths to pass tests --- datasets/bigbench/bigbench.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 86697b47344..477105f71c3 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -31,7 +31,8 @@ import bigbench.api.util as bb_utils except ModuleNotFoundError as error: - print(f"Failed to import bigbench. Please see https://github.com/google/BIG-bench or install using 'pip install git+https://github.com/google/BIG-bench.git'") + print("Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" + "using 'pip install git+https://github.com/google/BIG-bench.git'") raise _CITATION = """\ @@ -155,7 +156,7 @@ def _split_generators(self, dl_manager): return [ datasets.SplitGenerator( - name=datasets.splits.NamedSplit('default'), # TODO(ajandreassen): Is there a way override and call this 'all'? + name=datasets.splits.NamedSplit('default'), # TODO(ajandreassen): Is there a way to call this 'all'? # These kwargs will be passed to _generate_examples gen_kwargs={ "split": "all", From db422587251fe2fc85e26d2bc162b1bfed2f9881 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 16:24:32 -0700 Subject: [PATCH 10/91] reformatting using black --- datasets/bigbench/bigbench.py | 277 +++++++++++++++++++--------------- 1 file changed, 154 insertions(+), 123 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 477105f71c3..b14852ee82a 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -26,14 +26,16 @@ import datasets try: - import bigbench.bbseqio.bigbench_bridge as bbb - from bigbench.bbseqio import bigbench_json_paths as bb_json_paths - import bigbench.api.util as bb_utils - + import bigbench.bbseqio.bigbench_bridge as bbb + from bigbench.bbseqio import bigbench_json_paths as bb_json_paths + import bigbench.api.util as bb_utils + except ModuleNotFoundError as error: - print("Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" - "using 'pip install git+https://github.com/google/BIG-bench.git'") - raise + print( + "Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" + "using 'pip install git+https://github.com/google/BIG-bench.git'" + ) + raise _CITATION = """\ @InProceedings{bigbench, @@ -56,40 +58,51 @@ def div_or_none(x, y): - return x // y if x else x + return x // y if x else x def validate_task_name(task_name: str) -> None: - """Check that the requested task name is a valid bigbench json task.""" - if task_name in bb_utils.get_all_json_task_names(): - pass - elif task_name in bb_utils.get_all_programmatic_task_names(): - raise ValueError("BIG-Bench does not support programmatic tasks through HuggingFace datasets" - f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks.") - else: - print("Invalid task_name. Please choose one from:") - for name in bb_utils.get_all_json_task_names(): - print(f"-- {name}") - raise ValueError(f"Unknown task name. Got {task_name}.") - + """Check that the requested task name is a valid bigbench json task.""" + if task_name in bb_utils.get_all_json_task_names(): + pass + elif task_name in bb_utils.get_all_programmatic_task_names(): + raise ValueError( + "BIG-Bench does not support programmatic tasks through HuggingFace datasets" + f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks." + ) + else: + print("Invalid task_name. Please choose one from:") + for name in bb_utils.get_all_json_task_names(): + print(f"-- {name}") + raise ValueError(f"Unknown task name. Got {task_name}.") + + def validate_subtask_name(task_name: str, subtask_name: str) -> None: - """Check that the requested subtask name is a valid bigbench subtask.""" - subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] - if subtask_name not in subtasks: - print(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") - for name in subtasks: - print(f"-- {name}") - raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") - + """Check that the requested subtask name is a valid bigbench subtask.""" + subtasks = [ + name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name) + ] + if subtask_name not in subtasks: + print( + f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:" + ) + for name in subtasks: + print(f"-- {name}") + raise ValueError( + f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}." + ) + class BigBenchConfig(datasets.BuilderConfig): - def __init__(self, - task_name: str, - *args, - subtask_name: Optional[str] = None, - num_shots: int = 0, - max_examples: Optional[int] = None, - **kwargs): + def __init__( + self, + task_name: str, + *args, + subtask_name: Optional[str] = None, + num_shots: int = 0, + max_examples: Optional[int] = None, + **kwargs, + ): super().__init__( *args, **kwargs, @@ -101,18 +114,18 @@ def __init__(self, subtask_name: BIG-bench subtask name. Accepts both "task_name:subtask_name" and "subtask_name" formats. num_shots: Number of few-shot examples in input prompt. Default is zero. max_examples: Limit number of examples for each task. Default is including all examples. - """ + """ self.task_name = task_name self.subtask_name = subtask_name self.num_shots = num_shots self.max_examples = max_examples - + validate_task_name(self.task_name) if self.subtask_name: - # Subtasks are sometimes in bigbench written as task_name:subtask_name. - # We want to remove the task_name from the subtask names: - self.subtask_name = self.subtask_name.split(':')[-1] - validate_subtask_name(self.task_name, self.subtask_name) + # Subtasks are sometimes in bigbench written as task_name:subtask_name. + # We want to remove the task_name from the subtask names: + self.subtask_name = self.subtask_name.split(":")[-1] + validate_subtask_name(self.task_name, self.subtask_name) class BigBench(datasets.GeneratorBasedBuilder): @@ -123,16 +136,16 @@ class BigBench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - def _info(self): features = datasets.Features( { - "idx": datasets.Value("int32"), - "inputs": datasets.Value("string"), - "targets": datasets.Sequence(datasets.Value("string")), - "multiple_choice_targets": datasets.Sequence(datasets.Value("string")), - "multiple_choice_scores": datasets.Sequence(datasets.Value("int32")), - }) + "idx": datasets.Value("int32"), + "inputs": datasets.Value("string"), + "targets": datasets.Sequence(datasets.Value("string")), + "multiple_choice_targets": datasets.Sequence(datasets.Value("string")), + "multiple_choice_scores": datasets.Sequence(datasets.Value("int32")), + } + ) return datasets.DatasetInfo( # This is the description that will appear on the datasets page. description=_DESCRIPTION, @@ -150,17 +163,18 @@ def _info(self): citation=_CITATION, ) - def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - + return [ - datasets.SplitGenerator( - name=datasets.splits.NamedSplit('default'), # TODO(ajandreassen): Is there a way to call this 'all'? - # These kwargs will be passed to _generate_examples - gen_kwargs={ - "split": "all", - }, + datasets.SplitGenerator( + name=datasets.splits.NamedSplit( + "default" + ), # TODO(ajandreassen): Is there a way to call this 'all'? + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "split": "all", + }, ), datasets.SplitGenerator( name=datasets.Split.TRAIN, @@ -179,79 +193,96 @@ def _split_generators(self, dl_manager): ] def _generate_examples( - self, split # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + self, + split, # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` ): - """ Yields examples as (key, example) tuples. """ - if split == 'all': - # not cutoff in number of examples for 'all' split - MIN_VALIDATION_EXAMPLES = 0 + """Yields examples as (key, example) tuples.""" + if split == "all": + # not cutoff in number of examples for 'all' split + MIN_VALIDATION_EXAMPLES = 0 else: - MIN_VALIDATION_EXAMPLES = 16 + MIN_VALIDATION_EXAMPLES = 16 try: - task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) + task_path, json_util = bb_json_paths.get_task_path(self.config.task_name) - has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) - if has_subtasks: - subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) - num_subtasks = len(subtask_names) - min_validation_examples_per_subtask = div_or_none( - MIN_VALIDATION_EXAMPLES, num_subtasks) + has_subtasks = bb_json_paths.has_subtasks(self.config.task_name) + if has_subtasks: + subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) + num_subtasks = len(subtask_names) + min_validation_examples_per_subtask = div_or_none( + MIN_VALIDATION_EXAMPLES, num_subtasks + ) - if not has_subtasks: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = None, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, - min_validation_examples = MIN_VALIDATION_EXAMPLES - ) - ds_list = [ds_fn(split)] - elif self.config.subtask_name is not None: - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = self.config.subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = self.config.max_examples, - json_util = json_util, - min_validation_examples = min_validation_examples_per_subtask) - ds_list = [ds_fn(split)] - else: - # Create mixture of all subtasks - ds_list = [] - for subtask_name in subtask_names: - subtask_name = subtask_name.split(':')[-1] - print(f"Loading subtask {split} split", subtask_name) - ds_fn = bbb.get_dataset_fn(task_name = self.config.task_name, - task_path = task_path, - subtask_name = subtask_name, - num_shots = self.config.num_shots, - bigbench_task_type = bbb.BigBenchTaskType.HUGGINGFACE, - max_examples = div_or_none(self.config.max_examples, num_subtasks), - json_util = json_util, - min_validation_examples = min_validation_examples_per_subtask - ) - ds_list.append(ds_fn(split)) + if not has_subtasks: + ds_fn = bbb.get_dataset_fn( + task_name=self.config.task_name, + task_path=task_path, + subtask_name=None, + num_shots=self.config.num_shots, + bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, + max_examples=self.config.max_examples, + json_util=json_util, + min_validation_examples=MIN_VALIDATION_EXAMPLES, + ) + ds_list = [ds_fn(split)] + elif self.config.subtask_name is not None: + ds_fn = bbb.get_dataset_fn( + task_name=self.config.task_name, + task_path=task_path, + subtask_name=self.config.subtask_name, + num_shots=self.config.num_shots, + bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, + max_examples=self.config.max_examples, + json_util=json_util, + min_validation_examples=min_validation_examples_per_subtask, + ) + ds_list = [ds_fn(split)] + else: + # Create mixture of all subtasks + ds_list = [] + for subtask_name in subtask_names: + subtask_name = subtask_name.split(":")[-1] + print(f"Loading subtask {split} split", subtask_name) + ds_fn = bbb.get_dataset_fn( + task_name=self.config.task_name, + task_path=task_path, + subtask_name=subtask_name, + num_shots=self.config.num_shots, + bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, + max_examples=div_or_none( + self.config.max_examples, num_subtasks + ), + json_util=json_util, + min_validation_examples=min_validation_examples_per_subtask, + ) + ds_list.append(ds_fn(split)) except ValueError as value_error: - # BIG-Bench requires at least 16 examples to use the train & validation splits, - # while using 'all'/'default' does not have such a requirement. - if "has too few examples" in value_error.args[0] and split != 'all': - print(f"-- WARNING: skipping split {split} because it has too few examples. Please use 'default' split.") - print(value_error) - return - raise value_error - + # BIG-Bench requires at least 16 examples to use the train & validation splits, + # while using 'all'/'default' does not have such a requirement. + if "has too few examples" in value_error.args[0] and split != "all": + print( + f"-- WARNING: skipping split {split} because it has too few examples. Please use 'default' split." + ) + print(value_error) + return + raise value_error + unique_key_counter = 0 for ds in ds_list: - for example in ds: - unique_key_counter +=1 - yield unique_key_counter, { - "idx": example["idx"], - "inputs": example["inputs"].numpy().decode().strip(), - "targets": [target.numpy().decode().strip() for target in example["targets"]], - "multiple_choice_targets": [targets.decode().strip() for targets in example["multiple_choice_targets"].numpy()], - "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], - } + for example in ds: + unique_key_counter += 1 + yield unique_key_counter, { + "idx": example["idx"], + "inputs": example["inputs"].numpy().decode().strip(), + "targets": [ + target.numpy().decode().strip() for target in example["targets"] + ], + "multiple_choice_targets": [ + targets.decode().strip() + for targets in example["multiple_choice_targets"].numpy() + ], + "multiple_choice_scores": [ + scores for scores in example["multiple_choice_scores"].numpy() + ], + } From ca3693add54e4de46fdf6df69de0ec1113d5a74c Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 16:27:00 -0700 Subject: [PATCH 11/91] reformatting using black with py36 --- datasets/bigbench/bigbench.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index b14852ee82a..c1dfb11cec9 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -79,18 +79,12 @@ def validate_task_name(task_name: str) -> None: def validate_subtask_name(task_name: str, subtask_name: str) -> None: """Check that the requested subtask name is a valid bigbench subtask.""" - subtasks = [ - name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name) - ] + subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] if subtask_name not in subtasks: - print( - f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:" - ) + print(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") for name in subtasks: print(f"-- {name}") - raise ValueError( - f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}." - ) + raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") class BigBenchConfig(datasets.BuilderConfig): @@ -168,9 +162,7 @@ def _split_generators(self, dl_manager): return [ datasets.SplitGenerator( - name=datasets.splits.NamedSplit( - "default" - ), # TODO(ajandreassen): Is there a way to call this 'all'? + name=datasets.splits.NamedSplit("default"), # TODO(ajandreassen): Is there a way to call this 'all'? # These kwargs will be passed to _generate_examples gen_kwargs={ "split": "all", @@ -210,9 +202,7 @@ def _generate_examples( if has_subtasks: subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) num_subtasks = len(subtask_names) - min_validation_examples_per_subtask = div_or_none( - MIN_VALIDATION_EXAMPLES, num_subtasks - ) + min_validation_examples_per_subtask = div_or_none(MIN_VALIDATION_EXAMPLES, num_subtasks) if not has_subtasks: ds_fn = bbb.get_dataset_fn( @@ -250,9 +240,7 @@ def _generate_examples( subtask_name=subtask_name, num_shots=self.config.num_shots, bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, - max_examples=div_or_none( - self.config.max_examples, num_subtasks - ), + max_examples=div_or_none(self.config.max_examples, num_subtasks), json_util=json_util, min_validation_examples=min_validation_examples_per_subtask, ) @@ -275,14 +263,9 @@ def _generate_examples( yield unique_key_counter, { "idx": example["idx"], "inputs": example["inputs"].numpy().decode().strip(), - "targets": [ - target.numpy().decode().strip() for target in example["targets"] - ], + "targets": [target.numpy().decode().strip() for target in example["targets"]], "multiple_choice_targets": [ - targets.decode().strip() - for targets in example["multiple_choice_targets"].numpy() - ], - "multiple_choice_scores": [ - scores for scores in example["multiple_choice_scores"].numpy() + targets.decode().strip() for targets in example["multiple_choice_targets"].numpy() ], + "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], } From bf27af2983dceea3e3344783edd94011a160b339 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 16:30:46 -0700 Subject: [PATCH 12/91] using isort to fix import order --- datasets/bigbench/bigbench.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index c1dfb11cec9..a38b885cf0d 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -19,16 +19,18 @@ import csv import json -import numpy as np import os from typing import Optional +import numpy as np + import datasets + try: + import bigbench.api.util as bb_utils import bigbench.bbseqio.bigbench_bridge as bbb from bigbench.bbseqio import bigbench_json_paths as bb_json_paths - import bigbench.api.util as bb_utils except ModuleNotFoundError as error: print( From 9f46efcd990f05c1a74f765899edfa8826ae6c00 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 16:36:39 -0700 Subject: [PATCH 13/91] formatting for flake8 --- datasets/bigbench/bigbench.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index a38b885cf0d..ed5b6acf73b 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -17,13 +17,8 @@ """ -import csv -import json -import os from typing import Optional -import numpy as np - import datasets @@ -37,6 +32,7 @@ "Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" "using 'pip install git+https://github.com/google/BIG-bench.git'" ) + print(error) raise _CITATION = """\ @@ -50,7 +46,7 @@ """ _DESCRIPTION = """\ -The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to +The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models, and extrapolate their future capabilities. """ @@ -103,8 +99,8 @@ def __init__( *args, **kwargs, ) - """BIG-bench configuration. - + """BIG-bench configuration. + Args: task_name: BIG-bench task name. subtask_name: BIG-bench subtask name. Accepts both "task_name:subtask_name" and "subtask_name" formats. From 36460a1113755e3b18744f8193567a66775da6da Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 17:34:49 -0700 Subject: [PATCH 14/91] updated readme --- datasets/bigbench/README.md | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 2adb75357a3..a1d377134ac 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -58,15 +58,49 @@ To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo ### Data Instances -[More Information Needed] +Each dataset contains 5 features. For example an instance from the `emoji_movie` task is: + +- `idx`: + + 0 +- `inputs`: + + Q: What movie does this emoji describe? 👦👓⚡️ + choice: harry potter + choice: shutter island + choice: inglourious basterds + choice: die hard + choice: moonlight + A: + +- `targets`: + + ['harry potter'] +- `multiple_choice_targets`: + + ['harry potter', 'shutter island', 'die hard', 'inglourious basterds', 'moonlight'] + +- `multiple_choice_scores`: + + [1, 0, 0, 0, 0] + +For tasks that do not have multiple choice targets, the lists are empty. + ### Data Fields -[More Information Needed] +Every example has the following fields + - `idx`: an `int` feature + - `inputs`: a `string` feature + - `targets`: a sequence of `string` feature + - `multiple_choice_targets`: a sequence of `string` features + - `multiple_choice_scores`: a sequence of `int` features ### Data Splits -[More Information Needed] +Each task has a `default`, `train` and `validation` split. +The split `default` uses all the samples for each task (and it's the same as `all` used in the `bigbench.bbseqio` implementation.) +For standard evaluation on BIG-bench, we recommend using the `default` split, and the `train` and `validation` split is to be used if one wants to train a model on BIG-bench. ## Dataset Creation @@ -78,7 +112,7 @@ To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo #### Initial Data Collection and Normalization -[More Information Needed] +Tasks were contributed by the research community through [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench), and [reviewed](https://github.com/google/BIG-bench/blob/main/docs/doc.md#submission-review-process) by members of the collaboration. #### Who are the source language producers? @@ -124,7 +158,7 @@ To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo ### Citation Information -[More Information Needed] +A paper is in progress. Until then, please cite the BIG-bench collaboration and the GitHub [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench). ### Contributions From 3d755dca2e37a0c0aaea0c8938c6a51c7dd083b4 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 17:42:23 -0700 Subject: [PATCH 15/91] added YAML tags --- datasets/bigbench/README.md | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index a1d377134ac..6fb95cd03cd 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -1,6 +1,45 @@ --- YAML tags: -- copy-paste the tags obtained with the tagging app: https://github.com/huggingface/datasets-tagging +annotations_creators: +- crowdsourced +- expert-generated +- machine-generated +language_creators: +- crowdsourced +- expert-generated +- machine-generated +- other +languages: [] +licenses: +- apache-2.0 +multilinguality: +- multilingual +- monolingual +pretty_name: bigbench +size_categories: +- unknown +source_datasets: +- original +task_categories: +- multiple-choice +- question-answering +- text-classification +- text-generation +- zero-shot-classification +- other +task_ids: +- multiple-choice-qa +- extractive-qa +- open-domain-qa +- closed-domain-qa +- fact-checking +- acceptability-classification +- intent-classification +- multi-class-classification +- multi-label-classification +- text-scoring +- hate-speech-detection +- language-modeling --- # Dataset Card for BIG-bench From 00ee938a1fed7db3313dd9a84839f2724b319cf2 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 17:45:27 -0700 Subject: [PATCH 16/91] Adding bigbench github to additional requirements to pass tests --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index a827c308c9f..08e6ef1d4ef 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -2,3 +2,4 @@ unbabel-comet>=1.0.0;python_version>'3.6' git+https://github.com/google-research/bleurt.git git+https://github.com/ns-moosavi/coval.git git+https://github.com/hendrycks/math.git +git+https://github.com/google/BIG-bench.git From adf89b9f8ff7ba655133e1f2e8c895566b80906a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 18:02:49 -0700 Subject: [PATCH 17/91] replaing print statement with logger --- datasets/bigbench/bigbench.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index ed5b6acf73b..f340ed0943b 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -21,6 +21,7 @@ import datasets +logger = datasets.logging.get_logger(__name__) try: import bigbench.api.util as bb_utils @@ -28,11 +29,11 @@ from bigbench.bbseqio import bigbench_json_paths as bb_json_paths except ModuleNotFoundError as error: - print( + logger.info( "Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" "using 'pip install git+https://github.com/google/BIG-bench.git'" ) - print(error) + logger.error(error) raise _CITATION = """\ @@ -69,9 +70,9 @@ def validate_task_name(task_name: str) -> None: f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks." ) else: - print("Invalid task_name. Please choose one from:") + logger.error("Invalid task_name. Please choose one from:") for name in bb_utils.get_all_json_task_names(): - print(f"-- {name}") + logger.info(f"-- {name}") raise ValueError(f"Unknown task name. Got {task_name}.") @@ -79,9 +80,9 @@ def validate_subtask_name(task_name: str, subtask_name: str) -> None: """Check that the requested subtask name is a valid bigbench subtask.""" subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] if subtask_name not in subtasks: - print(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") + logger.error(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") for name in subtasks: - print(f"-- {name}") + logger.info(f"-- {name}") raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") @@ -231,7 +232,7 @@ def _generate_examples( ds_list = [] for subtask_name in subtask_names: subtask_name = subtask_name.split(":")[-1] - print(f"Loading subtask {split} split", subtask_name) + logger.info(f"Loading subtask {split} split", subtask_name) ds_fn = bbb.get_dataset_fn( task_name=self.config.task_name, task_path=task_path, @@ -247,10 +248,10 @@ def _generate_examples( # BIG-Bench requires at least 16 examples to use the train & validation splits, # while using 'all'/'default' does not have such a requirement. if "has too few examples" in value_error.args[0] and split != "all": - print( + logger.warning( f"-- WARNING: skipping split {split} because it has too few examples. Please use 'default' split." ) - print(value_error) + logger.warning(value_error) return raise value_error From 1c3128bcaa42d0b2737f6adee162a86188bff6bf Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 18:09:00 -0700 Subject: [PATCH 18/91] isort formatting --- datasets/bigbench/bigbench.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index f340ed0943b..3384c37cb86 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -21,6 +21,7 @@ import datasets + logger = datasets.logging.get_logger(__name__) try: From aab36f6761886281dd6c3c985246a9c26004fd7a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 18:09:48 -0700 Subject: [PATCH 19/91] adding missing t5 dependency to additional requirements to pass tests --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 08e6ef1d4ef..c3d50da2f0a 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -3,3 +3,4 @@ git+https://github.com/google-research/bleurt.git git+https://github.com/ns-moosavi/coval.git git+https://github.com/hendrycks/math.git git+https://github.com/google/BIG-bench.git +t5 From c2364d7952579df8a08e982bc3dbc9d79eb30855 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 18:25:40 -0700 Subject: [PATCH 20/91] adding missing seqio dependency to additional requirements to pass tests --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index c3d50da2f0a..3948a81f518 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -4,3 +4,4 @@ git+https://github.com/ns-moosavi/coval.git git+https://github.com/hendrycks/math.git git+https://github.com/google/BIG-bench.git t5 +seqio From ba07c2781735434ec5c2601606e12e41c6999402 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 18:53:55 -0700 Subject: [PATCH 21/91] adding missing tfds dependency to additional requirements to pass tests --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 3948a81f518..7b8fe81df01 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -5,3 +5,4 @@ git+https://github.com/hendrycks/math.git git+https://github.com/google/BIG-bench.git t5 seqio +tensorflow_datasets From 4f5f9613c493e0b88c44c1febb5df5ac00cf3c16 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 7 Apr 2022 19:09:49 -0700 Subject: [PATCH 22/91] adding missing tf metadata dependency to additional requirements to pass tests --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 7b8fe81df01..f8b38570ac9 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -6,3 +6,4 @@ git+https://github.com/google/BIG-bench.git t5 seqio tensorflow_datasets +tensorflow_metadata From b602a7972fdfbedacd304e36d216955e2ad326ad Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 11:07:00 -0700 Subject: [PATCH 23/91] Edits to make tests pass. Some modifications to args (task_name -> name), some edits to logging errors --- datasets/bigbench/bigbench.py | 55 +++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 3384c37cb86..5b9fe857d30 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -71,34 +71,38 @@ def validate_task_name(task_name: str) -> None: f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks." ) else: - logger.error("Invalid task_name. Please choose one from:") - for name in bb_utils.get_all_json_task_names(): - logger.info(f"-- {name}") + logger.error( + "Invalid task_name. Please choose one from:\n -- " + + "\n -- ".join(bb_utils.get_all_json_task_names()) + ) raise ValueError(f"Unknown task name. Got {task_name}.") def validate_subtask_name(task_name: str, subtask_name: str) -> None: """Check that the requested subtask name is a valid bigbench subtask.""" - subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] + subtasks = [ + name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name) + ] if subtask_name not in subtasks: - logger.error(f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:") - for name in subtasks: - logger.info(f"-- {name}") - raise ValueError(f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}.") + logger.error( + f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " + + "\n -- ".join(subtasks) + ) + raise ValueError( + f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}." + ) class BigBenchConfig(datasets.BuilderConfig): def __init__( self, - task_name: str, - *args, + name, subtask_name: Optional[str] = None, num_shots: int = 0, max_examples: Optional[int] = None, **kwargs, ): - super().__init__( - *args, + super().__init__(name=name, **kwargs, ) """BIG-bench configuration. @@ -109,7 +113,7 @@ def __init__( num_shots: Number of few-shot examples in input prompt. Default is zero. max_examples: Limit number of examples for each task. Default is including all examples. """ - self.task_name = task_name + self.task_name = name self.subtask_name = subtask_name self.num_shots = num_shots self.max_examples = max_examples @@ -130,6 +134,8 @@ class BigBench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig + BUILDER_CONFIGS = [BigBenchConfig(name='emoji_movie')]# for name in bb_utils.get_all_json_task_names()] + def _info(self): features = datasets.Features( { @@ -162,7 +168,9 @@ def _split_generators(self, dl_manager): return [ datasets.SplitGenerator( - name=datasets.splits.NamedSplit("default"), # TODO(ajandreassen): Is there a way to call this 'all'? + name=datasets.splits.NamedSplit( + "default" + ), # TODO(ajandreassen): Is there a way to call this 'all'? # These kwargs will be passed to _generate_examples gen_kwargs={ "split": "all", @@ -202,7 +210,9 @@ def _generate_examples( if has_subtasks: subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) num_subtasks = len(subtask_names) - min_validation_examples_per_subtask = div_or_none(MIN_VALIDATION_EXAMPLES, num_subtasks) + min_validation_examples_per_subtask = div_or_none( + MIN_VALIDATION_EXAMPLES, num_subtasks + ) if not has_subtasks: ds_fn = bbb.get_dataset_fn( @@ -240,7 +250,9 @@ def _generate_examples( subtask_name=subtask_name, num_shots=self.config.num_shots, bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, - max_examples=div_or_none(self.config.max_examples, num_subtasks), + max_examples=div_or_none( + self.config.max_examples, num_subtasks + ), json_util=json_util, min_validation_examples=min_validation_examples_per_subtask, ) @@ -263,9 +275,14 @@ def _generate_examples( yield unique_key_counter, { "idx": example["idx"], "inputs": example["inputs"].numpy().decode().strip(), - "targets": [target.numpy().decode().strip() for target in example["targets"]], + "targets": [ + target.numpy().decode().strip() for target in example["targets"] + ], "multiple_choice_targets": [ - targets.decode().strip() for targets in example["multiple_choice_targets"].numpy() + targets.decode().strip() + for targets in example["multiple_choice_targets"].numpy() + ], + "multiple_choice_scores": [ + scores for scores in example["multiple_choice_scores"].numpy() ], - "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], } From b4ff98775b79f4cb8263b04783ae1acf0a48da4e Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 11:44:11 -0700 Subject: [PATCH 24/91] fixed configs and moved checks for subtask and task names --- datasets/bigbench/bigbench.py | 38 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 5b9fe857d30..bda272952aa 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -83,32 +83,38 @@ def validate_subtask_name(task_name: str, subtask_name: str) -> None: subtasks = [ name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name) ] - if subtask_name not in subtasks: + if not subtasks: + raise ValueError( + f"Task {task_name} has no subtasks. Got subtask_name {subtask_name}." + ) + elif subtask_name not in subtasks: logger.error( f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " + "\n -- ".join(subtasks) ) raise ValueError( - f"Unknown subtask name. Got subtask {subtask_name} for task {task_name}." + f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " + + "\n -- ".join(subtasks) ) class BigBenchConfig(datasets.BuilderConfig): def __init__( self, - name, + name, subtask_name: Optional[str] = None, num_shots: int = 0, max_examples: Optional[int] = None, **kwargs, ): - super().__init__(name=name, + super().__init__( + name=name, **kwargs, ) """BIG-bench configuration. Args: - task_name: BIG-bench task name. + name: BIG-bench task name. subtask_name: BIG-bench subtask name. Accepts both "task_name:subtask_name" and "subtask_name" formats. num_shots: Number of few-shot examples in input prompt. Default is zero. max_examples: Limit number of examples for each task. Default is including all examples. @@ -118,15 +124,8 @@ def __init__( self.num_shots = num_shots self.max_examples = max_examples - validate_task_name(self.task_name) - if self.subtask_name: - # Subtasks are sometimes in bigbench written as task_name:subtask_name. - # We want to remove the task_name from the subtask names: - self.subtask_name = self.subtask_name.split(":")[-1] - validate_subtask_name(self.task_name, self.subtask_name) - -class BigBench(datasets.GeneratorBasedBuilder): +class Bigbench(datasets.GeneratorBasedBuilder): """The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models, and extrapolate their future capabilities.""" @@ -134,7 +133,9 @@ class BigBench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - BUILDER_CONFIGS = [BigBenchConfig(name='emoji_movie')]# for name in bb_utils.get_all_json_task_names()] + BUILDER_CONFIGS = [ + BigBenchConfig(name=name) for name in bb_utils.get_all_json_task_names() + ] def _info(self): features = datasets.Features( @@ -165,7 +166,6 @@ def _info(self): def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - return [ datasets.SplitGenerator( name=datasets.splits.NamedSplit( @@ -196,6 +196,14 @@ def _generate_examples( self, split, # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` ): + validate_task_name(self.config.task_name) + if self.config.subtask_name: + print("validating subtask_names", self.config.subtask_name) + # Subtasks are sometimes in bigbench written as task_name:subtask_name. + # We want to remove the task_name from the subtask names: + self.config.subtask_name = self.config.subtask_name.split(":")[-1] + validate_subtask_name(self.config.task_name, self.config.subtask_name) + """Yields examples as (key, example) tuples.""" if split == "all": # not cutoff in number of examples for 'all' split From d615d7ffc831ada5b6fbf8ac743dbcca6c890a12 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 11:49:07 -0700 Subject: [PATCH 25/91] update error messages and some formatting --- datasets/bigbench/bigbench.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index bda272952aa..9f50d9b14db 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -64,18 +64,17 @@ def div_or_none(x, y): def validate_task_name(task_name: str) -> None: """Check that the requested task name is a valid bigbench json task.""" if task_name in bb_utils.get_all_json_task_names(): - pass + return elif task_name in bb_utils.get_all_programmatic_task_names(): raise ValueError( "BIG-Bench does not support programmatic tasks through HuggingFace datasets" f"Please see {_HOMEPAGE} for more information for how to interact with the programmatic tasks." ) else: - logger.error( - "Invalid task_name. Please choose one from:\n -- " + raise ValueError( + f"Invalid task_name. Got task_name = {task_name}. Please choose one from:\n -- " + "\n -- ".join(bb_utils.get_all_json_task_names()) ) - raise ValueError(f"Unknown task name. Got {task_name}.") def validate_subtask_name(task_name: str, subtask_name: str) -> None: @@ -85,13 +84,9 @@ def validate_subtask_name(task_name: str, subtask_name: str) -> None: ] if not subtasks: raise ValueError( - f"Task {task_name} has no subtasks. Got subtask_name {subtask_name}." + f"Task {task_name} has no subtasks. Got subtask_name = {subtask_name}." ) elif subtask_name not in subtasks: - logger.error( - f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " - + "\n -- ".join(subtasks) - ) raise ValueError( f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " + "\n -- ".join(subtasks) @@ -198,7 +193,6 @@ def _generate_examples( ): validate_task_name(self.config.task_name) if self.config.subtask_name: - print("validating subtask_names", self.config.subtask_name) # Subtasks are sometimes in bigbench written as task_name:subtask_name. # We want to remove the task_name from the subtask names: self.config.subtask_name = self.config.subtask_name.split(":")[-1] From 804aac4503cbbd1485b999089f5c8279d5cbc9d2 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 11:49:53 -0700 Subject: [PATCH 26/91] more test requirements --- additional-tests-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index f8b38570ac9..686857510d3 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -7,3 +7,7 @@ t5 seqio tensorflow_datasets tensorflow_metadata +promise +gin-config +babel +editdistance From 04a3fbf4e2d6678ce4822c295b528adb78cdc4bb Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 11:54:28 -0700 Subject: [PATCH 27/91] file formatting --- datasets/bigbench/bigbench.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 9f50d9b14db..b5a1bc9ee86 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -79,13 +79,9 @@ def validate_task_name(task_name: str) -> None: def validate_subtask_name(task_name: str, subtask_name: str) -> None: """Check that the requested subtask name is a valid bigbench subtask.""" - subtasks = [ - name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name) - ] + subtasks = [name.split(":")[-1] for name in bb_utils.get_subtask_names_from_task(task_name)] if not subtasks: - raise ValueError( - f"Task {task_name} has no subtasks. Got subtask_name = {subtask_name}." - ) + raise ValueError(f"Task {task_name} has no subtasks. Got subtask_name = {subtask_name}.") elif subtask_name not in subtasks: raise ValueError( f"Invalid subtask_name {subtask_name} for task {task_name}. Please choose one from:\n -- " @@ -128,9 +124,7 @@ class Bigbench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - BUILDER_CONFIGS = [ - BigBenchConfig(name=name) for name in bb_utils.get_all_json_task_names() - ] + BUILDER_CONFIGS = [BigBenchConfig(name=name) for name in bb_utils.get_all_json_task_names()] def _info(self): features = datasets.Features( @@ -163,9 +157,7 @@ def _split_generators(self, dl_manager): """Returns SplitGenerators.""" return [ datasets.SplitGenerator( - name=datasets.splits.NamedSplit( - "default" - ), # TODO(ajandreassen): Is there a way to call this 'all'? + name=datasets.splits.NamedSplit("default"), # TODO(ajandreassen): Is there a way to call this 'all'? # These kwargs will be passed to _generate_examples gen_kwargs={ "split": "all", @@ -212,9 +204,7 @@ def _generate_examples( if has_subtasks: subtask_names = bb_json_paths.get_subtask_names(self.config.task_name) num_subtasks = len(subtask_names) - min_validation_examples_per_subtask = div_or_none( - MIN_VALIDATION_EXAMPLES, num_subtasks - ) + min_validation_examples_per_subtask = div_or_none(MIN_VALIDATION_EXAMPLES, num_subtasks) if not has_subtasks: ds_fn = bbb.get_dataset_fn( @@ -252,9 +242,7 @@ def _generate_examples( subtask_name=subtask_name, num_shots=self.config.num_shots, bigbench_task_type=bbb.BigBenchTaskType.HUGGINGFACE, - max_examples=div_or_none( - self.config.max_examples, num_subtasks - ), + max_examples=div_or_none(self.config.max_examples, num_subtasks), json_util=json_util, min_validation_examples=min_validation_examples_per_subtask, ) @@ -277,14 +265,9 @@ def _generate_examples( yield unique_key_counter, { "idx": example["idx"], "inputs": example["inputs"].numpy().decode().strip(), - "targets": [ - target.numpy().decode().strip() for target in example["targets"] - ], + "targets": [target.numpy().decode().strip() for target in example["targets"]], "multiple_choice_targets": [ - targets.decode().strip() - for targets in example["multiple_choice_targets"].numpy() - ], - "multiple_choice_scores": [ - scores for scores in example["multiple_choice_scores"].numpy() + targets.decode().strip() for targets in example["multiple_choice_targets"].numpy() ], + "multiple_choice_scores": [scores for scores in example["multiple_choice_scores"].numpy()], } From b5b8363f3ce5217497f1c138e106ce85f80bba5c Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:09:03 -0700 Subject: [PATCH 28/91] adding dummy.zip --- datasets/bigbench/dummy.zip | Bin 0 -> 108751 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 datasets/bigbench/dummy.zip diff --git a/datasets/bigbench/dummy.zip b/datasets/bigbench/dummy.zip new file mode 100644 index 0000000000000000000000000000000000000000..7ebcca274803aaed2f5aa71668524989c6aba3ce GIT binary patch literal 108751 zcmd6w3)C!Cb>~0ia^?Mym-3RQfCyY(!UaW8(HTL!ydxlGb$5N;b-(VezE$1#yUl0< z@r9UV5`&5+#__5s4+TUtF?fxjMjjd`gb;Pi=s3}ptnrm(NaBjl>{I94wR?A0Rafs_ zE9G2E*ZK86=j^l3KId$@c;|JyFPXotdE+ftrvGK_Kh`fDw$vK+dgD{Cx^lOr zou7L9rp8a<@6onBmUdcq-7j}vT6cxF)6%BG|NlPO|GyO`%Uy4aPeRAtQX=&9)Q226 zD|Ff^8&26^<-YokO^pJ%1E3|lE*ZH;9(`uVMewIS{QdkQxwNrf%OCn_gN?BHd#b^A zU;Wr6cW(OhEco zcE-lDjyUS@z21AxtzX&xiE}o7@q`2KS>M@lVfW+`YR}2OF6c#@L$4QY4Fe&#=jLi@ z*d1M>=mfrlvk&Y>?Xc;0y(HK&3i?ec!^4XOH6V|lC+h^wL$a^zjk?3MYj~@%-*0!r zep~32N0w@8n7*)rvJ+VM&OUU|87E;g5u!X`Mn{A1hypbykdY@gqkb1W&>MmuCS89R zMtvbZ^29t%4NCIFVr3_=9!tM@*B^VW(1&5uOZ;BoMKSzwc{zxK{!j>*Fbqo7HK@<5 z*aB7YEI#-47LW7%gUUBsDlsEc;d~>;%*OvhVDS2SMEMhhnd!wZnvzhT&0p-{ufI z3VxaqE^OI&63MVCs z9wxOkFlkXgtLOy2!?Iu7@0$SCuIs~I*!AOZIFX+!$OGB@)^u8lnw_tzLHeROU+mBz zKr=`u?6*P>vL%U`kUUsktp>4tp?_S)pio$Y_|&TMPPi~d(lfzc*!N(SXrh0JR{gjs z92ZK$$r%e8?&oCO=eNWmH$N!*>}J&KfqVC&c+d%Y(!3yzNz-~7xHNC(t2*J!loIU) zLx0eL1?;E~tE^<`!vG>q3?7sHLItuKuqRg&cY-gustv+mco@N=Rt%P0by`nDlw37m z)d`f88SqC#aMYv|4!o@}t1Uwep;&g&VzWqBgO-N!a&;#VADZ>7mLIQpoe>PIUMm>F z>Or2GrIk~zs0Qvy<=<RmJ`GZ)FyAbXy{Ogla55H#rL@Y*SJl9zRaTj{6L4uxH;bW> zx6+SRA^;_zPs6ph<(< z7T;S9S3*<&BJAswnwr$oFr?9AR?!K36#cLz5bWqRx`8x%O3_cQs0JuSKPA#mkfrjm zVb}}I&a}`gseCLG(h#J4*_50U)M#gI1p7*!*%XQgF)Y&+!8mObw2VM<3#8iuFjG$<*7oh#}DN80{u z#^V9(eZtPXaGa#LYC=lGkEXF1K_}Re53U4bZ>!%O1zyh|=zVZXPQ#FVFjv$Gj`WRb zv2xv2WE8T@k=JlOE6WsAGU5bx$`6ITLDwWSN3l5Or2G&Q(=es{P>!S%1Szsfi|rE+ z{*mr|Ldet#HPbt-q>Gg`Y^eZZp1u>HX_aXU^SUGXaS)2dCbcv$X_Yyv=mb707+>-G zeZM7Lpi>ne>S(Z02+P%+Kt@TOUYNlCdNUZ9o}GB{C`m$JS~E^#T7j$v=)@*v{x@53 zC-{;}u7;4y>`1F{$R$xnLy%mOt2u#;;$bMPfirO(Zy0%PNMHJWZxBbYE!mL9o|J1W zk=B4cEsKxl8gPO&#rAyw+cgz7W(1J(rS;{MoQ5gI&AFmZaGVYcA*U1v_SB7(B_+^o zSjZ$bJSR$)(|9;X*a@Z-b++RGaQdcb20XL}QD+Bs80FjM6m=FWYY0=+IZxjS(6s5c z+6kMTbbG^tkypBhPMdDidK#$oqvfkQfs*pdgP`9Yg%C(VRiW<<%|3@P;Y9r)Pg8@G z^2)`^PGF^ZrP&FR#Dfw#I9Y==^Fpo^XXJ`%z)~)zMA`|mv`UPFLEsNzd>xFvG$$^^ zO0ngvqJ}AD?2B}r07`>tx>o}M6$Gq`xrzqUX*~^08cg$5oj^&0X)u7@kDvz!79k9P z^K|kePZ~_~G&M+RFfCSg0xK=~6F9^aHbZm7F6=Kyu)-2krEkQdd_!h(ruSke!bBk2S|nnJ)e{m`7V8$|XHlb0Tn5l&51j$EXx;Y#Jm zK)vJM!Hn%1-4-YKT&Hs!-SorZh*b@Xz^1zr=EyrinZ^w`{u+5u*A|$?#*LaJ4viamni{q=ZWJp!hIOr9r&msx z?XWtsj(M8(U@ca50xKmPAU+p|FNArS;~>J3*PexDV&B zOzDuf8pW-|1BdSQeCZ|q6zLbpY6z1Tmx?>Vm!`7p3f}~_nl-0bsbl0wYPix=Rw(QQ zQ<}=sC=y&Xgp#e%jW>#m-f1eERn!orsjNuX381u+3L$4OhGiLSw`y3vL@o_4j zF|S|c>1uFOMM}B46NoAIxE!oP^;|Uc`d#UBbTn%+F%3+LNOB~dAV_KDUIdSPg;0k) zv5k>E_qot>3Y03!)70>!k*!$S39Pis)bvLX-okb0UI0(1HKhj*DP*1z)BvU8&H`B{ zXp)n{^-)nHj3BTXn-!pAS1OKHmV@~`Y6ud4&xgi_~kCws7g8&r75=-04<`xdxnIP1{0A z)Px&Fp;T@FNnh#kOpBQ*ISo_F8Rd#P!IAO;$*SLhlQkYDpp^HROfp`9=xBRTqIl*<#Kr^DAU4f1XpWe1(&%nCasSH&avWH zE`Kfiu=3m8z6VPc12d8!7aGUA~6fNPHmPh|ArWwG?1(#?1Z(Wc}yXlee;6?K9m^|-*?Yz!OlaL3fdWe6UG z5hnExxnl7NCrSrbv7sSO>EP;?ocI}NZW?sPN&1Kr)cL`wGWjK66ny7MY9Q05T%oWN zOeYSa$37gwX>_9`iKNB4G?6x$nua9BlzEy?K%}(8)^O;=4!Ly5Eo zFQpyk8gPO&rE&99Zo)`{g46<84dZ!f!nLw?C-~A_m)20jDh@_hbN5Fl>ZgaQ{qCRx z2lUK7k@VSZn(M0A(2%FOuDT^Beg-O*fGX(_HkO*++EoYA=#4g0XB9PwDatR>bpj|2 zq<#aAl$qhV@5eE?H!Mfk`%N^0{NnAK8l!D{EbX-Jx?k?TwC)OTr=?97@4PPm33Te@ za}7-4e)Ju=W)4_dTH1BVytMwvqtEQP2>#TEzt7H=A{EW#(Es|Z!Mm@1?2b@*QI zz2?@hZ2!bL8^3tMf%mNM?6|Od@&ue~;H%+dv%e_J=yW=8;Z4}?n~F<9z0me3g&ACJ zife&pK357e@-%ZGe)T^bBE7)bg#9~djUPQ9U{y5G=-kbumJ_gO?aMEh7rGp+eW&C! z_-O5$E9wMC>TBjm{s^vEG#9J*g8}?WIu%iFEKgHIl%khnWhbys_MBGW!Pf?`MVU0? zFnwA<=%sXG&723FRn?%S6KiGKPQaz-4&fHq4y;6_n>#d!WI`H*lx>)jbAlSBSK!&w z!kDOFQq7NvsKGgs8j_S=DHL{sDFIkpRt7`3Jsp-)SgI`@LTTzK6xQ&a*xa=hC$rye z<(;5Bu~zX zxe`eCgXjlhVj7O*{5g_N5Tp#TDJp`sxU`gkGQ^pXh8(%}l$;aPCN{yMt)Y2t6W%g_ z=?8MK@|8BlEWJEY4NbamphVgUvJ*k2JxmqW5mb_th9dnyYlo=Jw^>0a*ij@{)4?eU z31$Q}9H|1OK-LMGG@zwdaJPcSsNI$)duYg*lGBi+&5vABCpc2W6LTsuDXKAAXg_VD zE9Qx6n9^98`@IhJcj}(aaM_Pf+u$|3Fo$a|zM-I{KvqMSBKA^oCp?!jzVM8my~1r6 zZG$Hu#b+C6H*Ho?LzObVMY>J^rC_t~Zw=e9`VlUCXl%}eH25gkoRV{b8o6ORGN;Rj z!61Efyc4Yo@sS(ODrz{A8y4w00hEe^;DXL(G>YL?6}T!8R*vQ%G*lQXf_OqurmcZ| zdbSNP-+~jkDPPwa_2AO%C~o`k(tCPYqcrnD`MNw!4PMIE6)QV|l?E*+d=521g$Zo)sFmI_tnd4Ini`zc_luRCz)E?~bca8Ib(MT&N_kHvrU6Myh#W~L2vT2M z_F?lV8AA@whj;Ge7%8ZlRn)Mgh@wc>37{0)r+07#y+}ApQf!|IY4FiBIVI-=HEL{Y z++2YN|LiTC(xH)ZEmLwDf)o$riaNoO4pOdNv18r=Hh09Yy@qYBROkjRsNaPDeYp!E zz7#P%F^ZMrhu-Q+u&+2-nTX^8)8)uoknX`GH=kvgyQ+MD<|i z>duJ}O*n*8mGiWo1~gSU=c_t_l0GtvCtyhyjfODrz)?lG6E|vXhPzv(!w(%kny0Tp zO@mxDD^8%NkPhx9f>EUFWuv~(k13>^Rn*|6kgiDA381vfNJfzLhI4drn5+m{(kg?A zX)w|%BS+E+g0z>k9K}8GNc+&4$>R0@$<5|ZUIwH z{1OysV~p*!6-Ad6Xiv#$=u)7aE9wMCO1w5X3k<| z4c&<)LgANNeJ4QE=I+{SCS%y67@g)4Px5dEfsfyFHLUg(>5?R`?2^O zKTU2-Ov93Xs2oWr2vTZ1YC%{F=k_7Y6-EaXyG?3o_)%(nR?!K3)MQ(|4oibUly2jU zg%BxRomJGJq;R!J*9o9h{+8AO!)OVw^sscerJaQwNexiydWFJHFr@((Fs4I; z*N7sx+rBTHz-Yjo5!CRc0k=T*Ka}Q9&pKjfdr8dn-Eixdp1XEW+`daczR>*F^uqw= zE=xNw-)RYKE5K(iL-1bH1p#a?gAQ{&gS|eJA{D`+=D#{z@R7b1U4Y zi_Q1u-#YjFHR+u|^rJWbO_B8EFI8qFAMkV7_iqJT_WNP85=uMLqY)x5&t-i@T|PAw(2@M2eVc@nAySg2R>mgLUcG2U%xXpVn$rf>3GKJ4f9OsU>(BRv-3J^ zkJ)7Q;24nat(yB{)^3MbysI6_^sgz^mm_;2Lw0`!D>`a>vsPWZ3sSRJgPWYj;-n5> zn%ieI<#di@C&m5fyl0oS!dlT$ zJC^Xf zj)m;$nDR;N^}At9VF$2lI4=7uu=WVH$v9$PvK;R&QJ1qjfw40|R!Zd5*%s8{gSb<4<`m}$8^8f;tqyk zx9yf6E0dSgI+6L-ujU)kA={53yJK%e))~XKNe#F+N#;{LJpOr}tQ^r%4AIN?$@yk0 zu0zA4f&Z16fheQ7l8#lK(eg4B%1u$;tR>YFHzUgIgI`Ow>}5ymAu}P8nE!kuvu-XyfjyyK6c6A6DYNaixNq{o7fdx~tx z$>ES{yXcJ-E$MviP}Yx*KMY&ff|KDMoCP+iq+kg9hPG03aeNr#TYPUSHy~$s66>Z< zS23dlw?6}S^Wg|?7!A@`uL*nCqN#wdeZP?}EC+=Hr**QUj>!G)I{N0G4A3KFDCnmT zaxCsjeEqT8^RrJ+73D~9OteU@%GK353tF(9;AfeItYFYoYvJrD@*2K2c-4`}Cf|ab z6<-@{uVhEZj?ad#JqlZ9!uAn6?69~;@!#cxh1zmnd^Y^2ITm!Jc&_I5qmdLuTagD@ zpx$UG-%{Z*-QO3B%USWQ%nObw{C;af2a69TKg^76fj!J{B{Z9vqK?Ru8K)nMjhNJx zBf@TOhpM-hsO#wPu;94kuoWzDhs}{jH+>Xj@t()Sf*Z>X$a(Rw;HfHRbl`Xe%5S^? z!41P6+_B?#MLxwVP`)-RDhGs_7wb?5A3yWGb@Vm9G4Wt#WCU+?K(Q(ueuW!|6k`)l z@a}d3HX=`7jtVzLEvjY(D>_%=Bgvj8VyiHl*b3;5>cu-CUpn1TqAn+f87WKbvvY0e zc=2BHh8JeMOm=#C)KAwivT6%H9NtqTEhmM!NlR*{lM24Qj?Tz8iawT^YR2OM>=MBa zq-c!d!N>_Lk*Z3sG?Xwx({WYD$jrES5+3@nQybSb0X_!>1xv^8z2F z6ow->fFSCcJfrZFd|^291PL+*Fi(LMPz|AWj>TF9Ds@Q?Ix2D{C0@re;vY}Vyo zdKxyT*oqt--^cx6Wm`H&<4d1QPRAD7(w*)|S}4r}mfLf6<*fM9=O0Scb#!>O=DW^7 zI&jxq-*3qlg}mnH$r(*K6%6C-%y`t9(_dRhzvD~)9hnXOYPA=S;1=fe6vX144#yDN z=?**#n^9mw&Wms7#MKPd=}KuC{aTs492YMrynntC9Wowmz5YBz){f2X7k}&Y z=sb?63HLf5TQh(MQ{cV8#GGsQ;UF+1oki;)?$Lg!zML3O6aHxhD>`a?m~OocslhRO zc-bm}v??56SgUjuVdJy)pOhPrbHk;+^|S*nC~Zd_9g{Ctc4P+QU=YB$^JX*{dugs! zL=EF8J0neMZU!cvQL%j%Zc%UY}Z$owW5yx$d{S-XZ8{(&WDTm z%)yVapWeVGE=JwUbmB3#6Z7Yp7!*M=M$FBaVGKh0}R9X*tX8lTN< zLX{w#ZZH)lA*6sBAsl0qr8Rh{vB!VLM&%liv*V%0auqW=*W>A$eP4s^YQoDU-F6g* z!%k1O^5j#^4F$?_ROryQcYeB5U&n}VAYOGbGFrQ@1}4p<;}3*eAbf7QqgY%{3bRd? z)Gz0m(81zk@<%Q~SV0#~Ai&EG@!GwK!YfaBKI54(c{wni&v@NyOWI(K=#cT@|6ek5 zYdJD=bv~ps8eu=Jk(2dC>@7Cvf8C|nq#_$~czpQ(WJOClf@ssW?%LO3>)=@103r-O z9mWE42t0js!u&h%#C8C$Uy9E(pUJ9-A70p|obdT9p}S$zqOGmNXL&dOF?|E{@c>f0 zo(~QSxe?)X%e(mnmt{?ym6vnm-TeLYjp&?|Ply*@p0x>_)Pp^Ao3>2T;-UzTuf5(= zsxODe*IqxZU`0m_SI4$%-+sj;HQu#NPxg{g16~hWkqweqLT+vSYh_zSD|gV_0(*2;q_AR2u>pq-*9;f!r%--63ZEm$mw;rTt2|2Z6e+}D{ zr(YMfU$0vF!ij-I$Li(+$1B zRSpLGwgq!dp|;Ltc4u6k$*i=~_Y@Wm$=JpF^@;X3MyF(rVO2)lLWJYB52@)(Wv;owEb1cY_ zVeD&NX}>p?x5V1fvEv0o4`mi1TWhNIys#BULige`y70GbeM3w zvktJXfo;LVHuec4xMopx7@jY)Zm3{I4v$y6?5J!@#}R)r%dyu)j#+JSeWgPf;8;^n zR5c|h%FhOz)~eXHI(ji5$^R}h4ZF#Mbn^w*k+O+`FH+A5uw^;&a(p}y^n3G-=sb~^ zcWhk7HbI4{&6=glMQN=&&nCaCL|qPymv{VNt_>Y8zD06t8+k#`_1iIAbPu_T^aXSI zG8+R1+he~!&xD*D-+n*8Gp8+e^ibY~{vk67wQFgjXRSJ3SJBKz*H?ttu0kVnggno) zy{f4?eQ?ueY+oM=bIm0mo_Sb3@xUE>JzsXYEcd~W&odzh$bE3#%AB^;;e&jZxGyug zWfUao(WnIW2jy4!a2MaXi!I8Lmt*6z#P#!y=sb?k66<={rZnyYM+@P7s?m)%3M)?e zEOB*-x*QmvB|bIRhK?7H`LF6DuMpO$V^~bU&QS=}TJkvpGctCLxU1ZNoEip?c4@Io zRN9U@`X(lSZM*Kx48p_HX=&TUmG@o{$Lf2xF#Nan_8(vatJsrcJ(umYyjfsX=bpU7 zy?6_@I(u5Q1`p+%{J%5ThMXpDxLYUvKNT(M2=Zjq`(q@y9IQeCPBirT-F6Mhs2}DF z%UR*PZdq+ia=*Kdj*Qtr+p)fr8R|vw=0XUUNGtQ*JQH#15F3)GFUN($k;V1S3RZM( z#k2UQkFZtdQF|z3=(oH89;|4}j=}Jy%Ug=Y<-B-x*SF`H(81y^cGK1jRy>XxVFae6t5SVAGVWrhtd_N+j*f`|x9yluXZB{xlhPW7tVXHXiNdDrM9~>+ z|Kj{@*u;t^{R&~zEhx&`&fvvVLt)|7BLpUuVy{v=>ptqOF z%Ng`yCcgg`MAnRbC{J&6qa=xXqOp+AS3fFHmV?3|%=X^%-kJa9b#z|7 z)cxDc($;Y3_h1Hri?EwR{YxW!sk`x3Y*2|AIX%AA{XlhNI(Or#y4So5+nAp*5&0bk zWOk_i@H`W8gqU!$4*08ThIIJ&G;-H(A-=Q((wH9ZxKm8L0?*+EXT$t=R=POWAG5d^%TG$AT}&;@?LW{3=zE!|(;!mx`q2boheopxX+*y^e0ho&WQhDRTyM z1lKf~Q|$g=0DqE=@;sZe_j|A@dHQl-7!O!?>Qu0zb1rmWJNWMRUTl>)as;oJ_XF7N zNt$su5U;WLNsRU!3vzP&B*r6^?C99>r(ycg?+dnreDkp@iUF%!x zZ=dLmkL?oBlO3x3+`|hH$Yc zECR49Q#O3zNNmAf_5o~Dp%FPgz72L?Ra3go$V2=a{s7x&GewX?Un`lF(c>ZheI@F0 zY&^u@^$&|%P>0j;v!?fE)=YrkZ$tKc2q*A`moxE6<^L6G%UR*MD?5~3^+9Ac{q=Q_ z;u(%#WVXz%s@cd-()=S_{FR7T`AAajiAihR)sgT3g!Nd2WyG*rEhd7Q*{&_(t~S(~@#B*fnf#{7}BI4#&O> z$9W$?91*0xd>GT0Ht=Wv#GQbkz8bAdhGE%(M%4_KbzM6Y^T(gYLj)OvuU^;a-pRr*^S;ooJr@ zacn~j7u`d;v?mOuIGW4zWa~onkz#Qj8s67l@d-o&*I{(WCNBf|PCx95)1JJqy{}YX z4vY7-pI5M=qlS}%9YH>DCsKp*sZId@k**2NliBj~e>`sb`8*SHc0BC7^sYH=v4(W` zcpl;xnMs?PF~yKDVzv5Emm$iW@-_Qw@5UD8$jhv@$Z_SFz!Jv~`FeiL! z=6mbtXnYUhtCw~6 zuC+@3u$EwIB~Mlk34aUgQ4bZ1>(KD1=lc5*O)Uok_{93jVsSYp+&{ON)_uD0`|Id{ zd;qyCGnU?^*$Nt?c3WDk&inSi6-moMVOH9Ly7U1AHTCUvVB&50$IMh>j^ZRW)G=V+ zu;bLLKZ9+_H6TaE$EodA%;-8J?=P1;i0x{I%YJ;?2CvzL*-v*z0M6hR-ACq`ki+AF z=+bBBw51Ne;|aWbGL!HoiM>c>7;S^saD-*SJc0MC5_LH=p1`~Ma|ms=1$FS^LDnxb zYkhxf*oK&06t43z@>PF~O~^#$ba;^Uo+4>oALLGS#-HH#w!SA&Q1PKA`@4uC5#v4nJ#Q?H3W%4BL^XFQ>&j<_9ZS(RECo=X%wbuvO`%X9BAM>2itZxqd%iSk4KfWb0KwER)v( z;+^A`hY`@S4~hC@tbFW}M;qTRQJ15_dD)^m{gL7p)X@!j$oj3!T6!Z%(2GPP6c1UC z{xUWp6P44!naR@GQY5W&BL4mRd$ioQVtgroLy@!`5YDO=&;#WL zbeQ;ooNK;{nAR>EFt3=I3tM{u9GY)>=J0JdXhD4z{O`+Mh+Jg@J3gRe!-g*jtjghD zz@pgG{(SKk*U@M3UIa_vXW3WGJ9uTU^$X!-zH}{mE-T5n;cM9b#VpJDzmoC)OO<_{ z@AEQ}_dJRu%pGFj_ZYi~hEHCD_;-cca*n(P@w~^1zP=7`MO)Tezn9s{o}Gq;L^LvQ zoWuDTxUw#4Y=*nnWMcys(py^>JdW*}XG=~JM++%=aH!&@+3KY+}pGzEYMzpcS5{sLQ8 z$)FtQd2AZlAu+4#xjew!_DyUz#+F`NTy@0*%s(%Zmc!!#=3)Q6;M?o)N1j~!v&>Wz z()R%*LSrZSa1Q=!Y`~}qyuF`9p-+m^F3ZBFXx3F%JMq)yCrR?qyO^3 z?upDCP)hr)*N7sxw6$NuV0ZZ6Vq0d!<+%7@H=Jjpihu30ehL11W~XO%`VgeP4*8$v GPyZjv!*?G5 literal 0 HcmV?d00001 From 9501673d33498ac0a9de4646892f71071ab576a4 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:09:56 -0700 Subject: [PATCH 29/91] additional requirements again --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 686857510d3..994270050d8 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -7,6 +7,7 @@ t5 seqio tensorflow_datasets tensorflow_metadata +tensorflow_text promise gin-config babel From 80ea48c00c51dbc9823edb3b089aa89fb74977a9 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:24:32 -0700 Subject: [PATCH 30/91] tensorflow hub requirement --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 994270050d8..aed7b459a9c 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -8,6 +8,7 @@ seqio tensorflow_datasets tensorflow_metadata tensorflow_text +tensorflow_hub promise gin-config babel From c948370a466c2340427190def1157f414569d510 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:25:14 -0700 Subject: [PATCH 31/91] removing dummy.zip file. wrong format --- datasets/bigbench/dummy.zip | Bin 108751 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 datasets/bigbench/dummy.zip diff --git a/datasets/bigbench/dummy.zip b/datasets/bigbench/dummy.zip deleted file mode 100644 index 7ebcca274803aaed2f5aa71668524989c6aba3ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 108751 zcmd6w3)C!Cb>~0ia^?Mym-3RQfCyY(!UaW8(HTL!ydxlGb$5N;b-(VezE$1#yUl0< z@r9UV5`&5+#__5s4+TUtF?fxjMjjd`gb;Pi=s3}ptnrm(NaBjl>{I94wR?A0Rafs_ zE9G2E*ZK86=j^l3KId$@c;|JyFPXotdE+ftrvGK_Kh`fDw$vK+dgD{Cx^lOr zou7L9rp8a<@6onBmUdcq-7j}vT6cxF)6%BG|NlPO|GyO`%Uy4aPeRAtQX=&9)Q226 zD|Ff^8&26^<-YokO^pJ%1E3|lE*ZH;9(`uVMewIS{QdkQxwNrf%OCn_gN?BHd#b^A zU;Wr6cW(OhEco zcE-lDjyUS@z21AxtzX&xiE}o7@q`2KS>M@lVfW+`YR}2OF6c#@L$4QY4Fe&#=jLi@ z*d1M>=mfrlvk&Y>?Xc;0y(HK&3i?ec!^4XOH6V|lC+h^wL$a^zjk?3MYj~@%-*0!r zep~32N0w@8n7*)rvJ+VM&OUU|87E;g5u!X`Mn{A1hypbykdY@gqkb1W&>MmuCS89R zMtvbZ^29t%4NCIFVr3_=9!tM@*B^VW(1&5uOZ;BoMKSzwc{zxK{!j>*Fbqo7HK@<5 z*aB7YEI#-47LW7%gUUBsDlsEc;d~>;%*OvhVDS2SMEMhhnd!wZnvzhT&0p-{ufI z3VxaqE^OI&63MVCs z9wxOkFlkXgtLOy2!?Iu7@0$SCuIs~I*!AOZIFX+!$OGB@)^u8lnw_tzLHeROU+mBz zKr=`u?6*P>vL%U`kUUsktp>4tp?_S)pio$Y_|&TMPPi~d(lfzc*!N(SXrh0JR{gjs z92ZK$$r%e8?&oCO=eNWmH$N!*>}J&KfqVC&c+d%Y(!3yzNz-~7xHNC(t2*J!loIU) zLx0eL1?;E~tE^<`!vG>q3?7sHLItuKuqRg&cY-gustv+mco@N=Rt%P0by`nDlw37m z)d`f88SqC#aMYv|4!o@}t1Uwep;&g&VzWqBgO-N!a&;#VADZ>7mLIQpoe>PIUMm>F z>Or2GrIk~zs0Qvy<=<RmJ`GZ)FyAbXy{Ogla55H#rL@Y*SJl9zRaTj{6L4uxH;bW> zx6+SRA^;_zPs6ph<(< z7T;S9S3*<&BJAswnwr$oFr?9AR?!K36#cLz5bWqRx`8x%O3_cQs0JuSKPA#mkfrjm zVb}}I&a}`gseCLG(h#J4*_50U)M#gI1p7*!*%XQgF)Y&+!8mObw2VM<3#8iuFjG$<*7oh#}DN80{u z#^V9(eZtPXaGa#LYC=lGkEXF1K_}Re53U4bZ>!%O1zyh|=zVZXPQ#FVFjv$Gj`WRb zv2xv2WE8T@k=JlOE6WsAGU5bx$`6ITLDwWSN3l5Or2G&Q(=es{P>!S%1Szsfi|rE+ z{*mr|Ldet#HPbt-q>Gg`Y^eZZp1u>HX_aXU^SUGXaS)2dCbcv$X_Yyv=mb707+>-G zeZM7Lpi>ne>S(Z02+P%+Kt@TOUYNlCdNUZ9o}GB{C`m$JS~E^#T7j$v=)@*v{x@53 zC-{;}u7;4y>`1F{$R$xnLy%mOt2u#;;$bMPfirO(Zy0%PNMHJWZxBbYE!mL9o|J1W zk=B4cEsKxl8gPO&#rAyw+cgz7W(1J(rS;{MoQ5gI&AFmZaGVYcA*U1v_SB7(B_+^o zSjZ$bJSR$)(|9;X*a@Z-b++RGaQdcb20XL}QD+Bs80FjM6m=FWYY0=+IZxjS(6s5c z+6kMTbbG^tkypBhPMdDidK#$oqvfkQfs*pdgP`9Yg%C(VRiW<<%|3@P;Y9r)Pg8@G z^2)`^PGF^ZrP&FR#Dfw#I9Y==^Fpo^XXJ`%z)~)zMA`|mv`UPFLEsNzd>xFvG$$^^ zO0ngvqJ}AD?2B}r07`>tx>o}M6$Gq`xrzqUX*~^08cg$5oj^&0X)u7@kDvz!79k9P z^K|kePZ~_~G&M+RFfCSg0xK=~6F9^aHbZm7F6=Kyu)-2krEkQdd_!h(ruSke!bBk2S|nnJ)e{m`7V8$|XHlb0Tn5l&51j$EXx;Y#Jm zK)vJM!Hn%1-4-YKT&Hs!-SorZh*b@Xz^1zr=EyrinZ^w`{u+5u*A|$?#*LaJ4viamni{q=ZWJp!hIOr9r&msx z?XWtsj(M8(U@ca50xKmPAU+p|FNArS;~>J3*PexDV&B zOzDuf8pW-|1BdSQeCZ|q6zLbpY6z1Tmx?>Vm!`7p3f}~_nl-0bsbl0wYPix=Rw(QQ zQ<}=sC=y&Xgp#e%jW>#m-f1eERn!orsjNuX381u+3L$4OhGiLSw`y3vL@o_4j zF|S|c>1uFOMM}B46NoAIxE!oP^;|Uc`d#UBbTn%+F%3+LNOB~dAV_KDUIdSPg;0k) zv5k>E_qot>3Y03!)70>!k*!$S39Pis)bvLX-okb0UI0(1HKhj*DP*1z)BvU8&H`B{ zXp)n{^-)nHj3BTXn-!pAS1OKHmV@~`Y6ud4&xgi_~kCws7g8&r75=-04<`xdxnIP1{0A z)Px&Fp;T@FNnh#kOpBQ*ISo_F8Rd#P!IAO;$*SLhlQkYDpp^HROfp`9=xBRTqIl*<#Kr^DAU4f1XpWe1(&%nCasSH&avWH zE`Kfiu=3m8z6VPc12d8!7aGUA~6fNPHmPh|ArWwG?1(#?1Z(Wc}yXlee;6?K9m^|-*?Yz!OlaL3fdWe6UG z5hnExxnl7NCrSrbv7sSO>EP;?ocI}NZW?sPN&1Kr)cL`wGWjK66ny7MY9Q05T%oWN zOeYSa$37gwX>_9`iKNB4G?6x$nua9BlzEy?K%}(8)^O;=4!Ly5Eo zFQpyk8gPO&rE&99Zo)`{g46<84dZ!f!nLw?C-~A_m)20jDh@_hbN5Fl>ZgaQ{qCRx z2lUK7k@VSZn(M0A(2%FOuDT^Beg-O*fGX(_HkO*++EoYA=#4g0XB9PwDatR>bpj|2 zq<#aAl$qhV@5eE?H!Mfk`%N^0{NnAK8l!D{EbX-Jx?k?TwC)OTr=?97@4PPm33Te@ za}7-4e)Ju=W)4_dTH1BVytMwvqtEQP2>#TEzt7H=A{EW#(Es|Z!Mm@1?2b@*QI zz2?@hZ2!bL8^3tMf%mNM?6|Od@&ue~;H%+dv%e_J=yW=8;Z4}?n~F<9z0me3g&ACJ zife&pK357e@-%ZGe)T^bBE7)bg#9~djUPQ9U{y5G=-kbumJ_gO?aMEh7rGp+eW&C! z_-O5$E9wMC>TBjm{s^vEG#9J*g8}?WIu%iFEKgHIl%khnWhbys_MBGW!Pf?`MVU0? zFnwA<=%sXG&723FRn?%S6KiGKPQaz-4&fHq4y;6_n>#d!WI`H*lx>)jbAlSBSK!&w z!kDOFQq7NvsKGgs8j_S=DHL{sDFIkpRt7`3Jsp-)SgI`@LTTzK6xQ&a*xa=hC$rye z<(;5Bu~zX zxe`eCgXjlhVj7O*{5g_N5Tp#TDJp`sxU`gkGQ^pXh8(%}l$;aPCN{yMt)Y2t6W%g_ z=?8MK@|8BlEWJEY4NbamphVgUvJ*k2JxmqW5mb_th9dnyYlo=Jw^>0a*ij@{)4?eU z31$Q}9H|1OK-LMGG@zwdaJPcSsNI$)duYg*lGBi+&5vABCpc2W6LTsuDXKAAXg_VD zE9Qx6n9^98`@IhJcj}(aaM_Pf+u$|3Fo$a|zM-I{KvqMSBKA^oCp?!jzVM8my~1r6 zZG$Hu#b+C6H*Ho?LzObVMY>J^rC_t~Zw=e9`VlUCXl%}eH25gkoRV{b8o6ORGN;Rj z!61Efyc4Yo@sS(ODrz{A8y4w00hEe^;DXL(G>YL?6}T!8R*vQ%G*lQXf_OqurmcZ| zdbSNP-+~jkDPPwa_2AO%C~o`k(tCPYqcrnD`MNw!4PMIE6)QV|l?E*+d=521g$Zo)sFmI_tnd4Ini`zc_luRCz)E?~bca8Ib(MT&N_kHvrU6Myh#W~L2vT2M z_F?lV8AA@whj;Ge7%8ZlRn)Mgh@wc>37{0)r+07#y+}ApQf!|IY4FiBIVI-=HEL{Y z++2YN|LiTC(xH)ZEmLwDf)o$riaNoO4pOdNv18r=Hh09Yy@qYBROkjRsNaPDeYp!E zz7#P%F^ZMrhu-Q+u&+2-nTX^8)8)uoknX`GH=kvgyQ+MD<|i z>duJ}O*n*8mGiWo1~gSU=c_t_l0GtvCtyhyjfODrz)?lG6E|vXhPzv(!w(%kny0Tp zO@mxDD^8%NkPhx9f>EUFWuv~(k13>^Rn*|6kgiDA381vfNJfzLhI4drn5+m{(kg?A zX)w|%BS+E+g0z>k9K}8GNc+&4$>R0@$<5|ZUIwH z{1OysV~p*!6-Ad6Xiv#$=u)7aE9wMCO1w5X3k<| z4c&<)LgANNeJ4QE=I+{SCS%y67@g)4Px5dEfsfyFHLUg(>5?R`?2^O zKTU2-Ov93Xs2oWr2vTZ1YC%{F=k_7Y6-EaXyG?3o_)%(nR?!K3)MQ(|4oibUly2jU zg%BxRomJGJq;R!J*9o9h{+8AO!)OVw^sscerJaQwNexiydWFJHFr@((Fs4I; z*N7sx+rBTHz-Yjo5!CRc0k=T*Ka}Q9&pKjfdr8dn-Eixdp1XEW+`daczR>*F^uqw= zE=xNw-)RYKE5K(iL-1bH1p#a?gAQ{&gS|eJA{D`+=D#{z@R7b1U4Y zi_Q1u-#YjFHR+u|^rJWbO_B8EFI8qFAMkV7_iqJT_WNP85=uMLqY)x5&t-i@T|PAw(2@M2eVc@nAySg2R>mgLUcG2U%xXpVn$rf>3GKJ4f9OsU>(BRv-3J^ zkJ)7Q;24nat(yB{)^3MbysI6_^sgz^mm_;2Lw0`!D>`a>vsPWZ3sSRJgPWYj;-n5> zn%ieI<#di@C&m5fyl0oS!dlT$ zJC^Xf zj)m;$nDR;N^}At9VF$2lI4=7uu=WVH$v9$PvK;R&QJ1qjfw40|R!Zd5*%s8{gSb<4<`m}$8^8f;tqyk zx9yf6E0dSgI+6L-ujU)kA={53yJK%e))~XKNe#F+N#;{LJpOr}tQ^r%4AIN?$@yk0 zu0zA4f&Z16fheQ7l8#lK(eg4B%1u$;tR>YFHzUgIgI`Ow>}5ymAu}P8nE!kuvu-XyfjyyK6c6A6DYNaixNq{o7fdx~tx z$>ES{yXcJ-E$MviP}Yx*KMY&ff|KDMoCP+iq+kg9hPG03aeNr#TYPUSHy~$s66>Z< zS23dlw?6}S^Wg|?7!A@`uL*nCqN#wdeZP?}EC+=Hr**QUj>!G)I{N0G4A3KFDCnmT zaxCsjeEqT8^RrJ+73D~9OteU@%GK353tF(9;AfeItYFYoYvJrD@*2K2c-4`}Cf|ab z6<-@{uVhEZj?ad#JqlZ9!uAn6?69~;@!#cxh1zmnd^Y^2ITm!Jc&_I5qmdLuTagD@ zpx$UG-%{Z*-QO3B%USWQ%nObw{C;af2a69TKg^76fj!J{B{Z9vqK?Ru8K)nMjhNJx zBf@TOhpM-hsO#wPu;94kuoWzDhs}{jH+>Xj@t()Sf*Z>X$a(Rw;HfHRbl`Xe%5S^? z!41P6+_B?#MLxwVP`)-RDhGs_7wb?5A3yWGb@Vm9G4Wt#WCU+?K(Q(ueuW!|6k`)l z@a}d3HX=`7jtVzLEvjY(D>_%=Bgvj8VyiHl*b3;5>cu-CUpn1TqAn+f87WKbvvY0e zc=2BHh8JeMOm=#C)KAwivT6%H9NtqTEhmM!NlR*{lM24Qj?Tz8iawT^YR2OM>=MBa zq-c!d!N>_Lk*Z3sG?Xwx({WYD$jrES5+3@nQybSb0X_!>1xv^8z2F z6ow->fFSCcJfrZFd|^291PL+*Fi(LMPz|AWj>TF9Ds@Q?Ix2D{C0@re;vY}Vyo zdKxyT*oqt--^cx6Wm`H&<4d1QPRAD7(w*)|S}4r}mfLf6<*fM9=O0Scb#!>O=DW^7 zI&jxq-*3qlg}mnH$r(*K6%6C-%y`t9(_dRhzvD~)9hnXOYPA=S;1=fe6vX144#yDN z=?**#n^9mw&Wms7#MKPd=}KuC{aTs492YMrynntC9Wowmz5YBz){f2X7k}&Y z=sb?63HLf5TQh(MQ{cV8#GGsQ;UF+1oki;)?$Lg!zML3O6aHxhD>`a?m~OocslhRO zc-bm}v??56SgUjuVdJy)pOhPrbHk;+^|S*nC~Zd_9g{Ctc4P+QU=YB$^JX*{dugs! zL=EF8J0neMZU!cvQL%j%Zc%UY}Z$owW5yx$d{S-XZ8{(&WDTm z%)yVapWeVGE=JwUbmB3#6Z7Yp7!*M=M$FBaVGKh0}R9X*tX8lTN< zLX{w#ZZH)lA*6sBAsl0qr8Rh{vB!VLM&%liv*V%0auqW=*W>A$eP4s^YQoDU-F6g* z!%k1O^5j#^4F$?_ROryQcYeB5U&n}VAYOGbGFrQ@1}4p<;}3*eAbf7QqgY%{3bRd? z)Gz0m(81zk@<%Q~SV0#~Ai&EG@!GwK!YfaBKI54(c{wni&v@NyOWI(K=#cT@|6ek5 zYdJD=bv~ps8eu=Jk(2dC>@7Cvf8C|nq#_$~czpQ(WJOClf@ssW?%LO3>)=@103r-O z9mWE42t0js!u&h%#C8C$Uy9E(pUJ9-A70p|obdT9p}S$zqOGmNXL&dOF?|E{@c>f0 zo(~QSxe?)X%e(mnmt{?ym6vnm-TeLYjp&?|Ply*@p0x>_)Pp^Ao3>2T;-UzTuf5(= zsxODe*IqxZU`0m_SI4$%-+sj;HQu#NPxg{g16~hWkqweqLT+vSYh_zSD|gV_0(*2;q_AR2u>pq-*9;f!r%--63ZEm$mw;rTt2|2Z6e+}D{ zr(YMfU$0vF!ij-I$Li(+$1B zRSpLGwgq!dp|;Ltc4u6k$*i=~_Y@Wm$=JpF^@;X3MyF(rVO2)lLWJYB52@)(Wv;owEb1cY_ zVeD&NX}>p?x5V1fvEv0o4`mi1TWhNIys#BULige`y70GbeM3w zvktJXfo;LVHuec4xMopx7@jY)Zm3{I4v$y6?5J!@#}R)r%dyu)j#+JSeWgPf;8;^n zR5c|h%FhOz)~eXHI(ji5$^R}h4ZF#Mbn^w*k+O+`FH+A5uw^;&a(p}y^n3G-=sb~^ zcWhk7HbI4{&6=glMQN=&&nCaCL|qPymv{VNt_>Y8zD06t8+k#`_1iIAbPu_T^aXSI zG8+R1+he~!&xD*D-+n*8Gp8+e^ibY~{vk67wQFgjXRSJ3SJBKz*H?ttu0kVnggno) zy{f4?eQ?ueY+oM=bIm0mo_Sb3@xUE>JzsXYEcd~W&odzh$bE3#%AB^;;e&jZxGyug zWfUao(WnIW2jy4!a2MaXi!I8Lmt*6z#P#!y=sb?k66<={rZnyYM+@P7s?m)%3M)?e zEOB*-x*QmvB|bIRhK?7H`LF6DuMpO$V^~bU&QS=}TJkvpGctCLxU1ZNoEip?c4@Io zRN9U@`X(lSZM*Kx48p_HX=&TUmG@o{$Lf2xF#Nan_8(vatJsrcJ(umYyjfsX=bpU7 zy?6_@I(u5Q1`p+%{J%5ThMXpDxLYUvKNT(M2=Zjq`(q@y9IQeCPBirT-F6Mhs2}DF z%UR*PZdq+ia=*Kdj*Qtr+p)fr8R|vw=0XUUNGtQ*JQH#15F3)GFUN($k;V1S3RZM( z#k2UQkFZtdQF|z3=(oH89;|4}j=}Jy%Ug=Y<-B-x*SF`H(81y^cGK1jRy>XxVFae6t5SVAGVWrhtd_N+j*f`|x9yluXZB{xlhPW7tVXHXiNdDrM9~>+ z|Kj{@*u;t^{R&~zEhx&`&fvvVLt)|7BLpUuVy{v=>ptqOF z%Ng`yCcgg`MAnRbC{J&6qa=xXqOp+AS3fFHmV?3|%=X^%-kJa9b#z|7 z)cxDc($;Y3_h1Hri?EwR{YxW!sk`x3Y*2|AIX%AA{XlhNI(Or#y4So5+nAp*5&0bk zWOk_i@H`W8gqU!$4*08ThIIJ&G;-H(A-=Q((wH9ZxKm8L0?*+EXT$t=R=POWAG5d^%TG$AT}&;@?LW{3=zE!|(;!mx`q2boheopxX+*y^e0ho&WQhDRTyM z1lKf~Q|$g=0DqE=@;sZe_j|A@dHQl-7!O!?>Qu0zb1rmWJNWMRUTl>)as;oJ_XF7N zNt$su5U;WLNsRU!3vzP&B*r6^?C99>r(ycg?+dnreDkp@iUF%!x zZ=dLmkL?oBlO3x3+`|hH$Yc zECR49Q#O3zNNmAf_5o~Dp%FPgz72L?Ra3go$V2=a{s7x&GewX?Un`lF(c>ZheI@F0 zY&^u@^$&|%P>0j;v!?fE)=YrkZ$tKc2q*A`moxE6<^L6G%UR*MD?5~3^+9Ac{q=Q_ z;u(%#WVXz%s@cd-()=S_{FR7T`AAajiAihR)sgT3g!Nd2WyG*rEhd7Q*{&_(t~S(~@#B*fnf#{7}BI4#&O> z$9W$?91*0xd>GT0Ht=Wv#GQbkz8bAdhGE%(M%4_KbzM6Y^T(gYLj)OvuU^;a-pRr*^S;ooJr@ zacn~j7u`d;v?mOuIGW4zWa~onkz#Qj8s67l@d-o&*I{(WCNBf|PCx95)1JJqy{}YX z4vY7-pI5M=qlS}%9YH>DCsKp*sZId@k**2NliBj~e>`sb`8*SHc0BC7^sYH=v4(W` zcpl;xnMs?PF~yKDVzv5Emm$iW@-_Qw@5UD8$jhv@$Z_SFz!Jv~`FeiL! z=6mbtXnYUhtCw~6 zuC+@3u$EwIB~Mlk34aUgQ4bZ1>(KD1=lc5*O)Uok_{93jVsSYp+&{ON)_uD0`|Id{ zd;qyCGnU?^*$Nt?c3WDk&inSi6-moMVOH9Ly7U1AHTCUvVB&50$IMh>j^ZRW)G=V+ zu;bLLKZ9+_H6TaE$EodA%;-8J?=P1;i0x{I%YJ;?2CvzL*-v*z0M6hR-ACq`ki+AF z=+bBBw51Ne;|aWbGL!HoiM>c>7;S^saD-*SJc0MC5_LH=p1`~Ma|ms=1$FS^LDnxb zYkhxf*oK&06t43z@>PF~O~^#$ba;^Uo+4>oALLGS#-HH#w!SA&Q1PKA`@4uC5#v4nJ#Q?H3W%4BL^XFQ>&j<_9ZS(RECo=X%wbuvO`%X9BAM>2itZxqd%iSk4KfWb0KwER)v( z;+^A`hY`@S4~hC@tbFW}M;qTRQJ15_dD)^m{gL7p)X@!j$oj3!T6!Z%(2GPP6c1UC z{xUWp6P44!naR@GQY5W&BL4mRd$ioQVtgroLy@!`5YDO=&;#WL zbeQ;ooNK;{nAR>EFt3=I3tM{u9GY)>=J0JdXhD4z{O`+Mh+Jg@J3gRe!-g*jtjghD zz@pgG{(SKk*U@M3UIa_vXW3WGJ9uTU^$X!-zH}{mE-T5n;cM9b#VpJDzmoC)OO<_{ z@AEQ}_dJRu%pGFj_ZYi~hEHCD_;-cca*n(P@w~^1zP=7`MO)Tezn9s{o}Gq;L^LvQ zoWuDTxUw#4Y=*nnWMcys(py^>JdW*}XG=~JM++%=aH!&@+3KY+}pGzEYMzpcS5{sLQ8 z$)FtQd2AZlAu+4#xjew!_DyUz#+F`NTy@0*%s(%Zmc!!#=3)Q6;M?o)N1j~!v&>Wz z()R%*LSrZSa1Q=!Y`~}qyuF`9p-+m^F3ZBFXx3F%JMq)yCrR?qyO^3 z?upDCP)hr)*N7sxw6$NuV0ZZ6Vq0d!<+%7@H=Jjpihu30ehL11W~XO%`VgeP4*8$v GPyZjv!*?G5 From 0e035e55f31754353e4476f7eb35109aad11ff64 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:27:22 -0700 Subject: [PATCH 32/91] adding individual dummy_data.zip for each config --- .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/anachronisms/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../analogical_similarity/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../analytic_entailment/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/arithmetic/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../ascii_word_recognition/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../authorship_verification/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../auto_categorization/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/auto_debugging/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/bbq_lite_json/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/causal_judgment/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/cause_and_effect/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/checkmate_in_one/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../chess_state_tracking/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../cifar10_classification/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../code_line_description/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/codenames/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/color/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/common_morpheme/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../conceptual_combinations/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../conlang_translation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/crash_blossom/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/crass_ai/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../cryobiology_spanish/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/cryptonite/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/cs_algorithms/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dark_humor_detection/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/date_understanding/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/disambiguation_qa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/dyck_languages/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/elementary_math_qa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/emoji_movie/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../empirical_judgments/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/english_proverbs/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../english_russian_proverbs/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/entailed_polarity/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../entailed_polarity_hindi/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../epistemic_reasoning/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/fact_checker/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/fantasy_reasoning/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/few_shot_nlg/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/general_knowledge/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/geometric_shapes/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/goal_step_wikihow/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/hhh_alignment/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../hindi_question_answering/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/hindu_knowledge/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/hinglish_toxicity/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../human_organs_senses/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/hyperbaton/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../identify_math_theorems/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../identify_odd_metaphor/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/implicatures/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/implicit_relations/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/intent_recognition/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/intersect_geometry/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../irony_identification/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/kanji_ascii/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/kannada/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/key_value_maps/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/known_unknowns/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/language_games/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../language_identification/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../linguistic_mappings/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../linguistics_puzzles/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/list_functions/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/logic_grid_puzzle/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/logical_args/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/logical_deduction/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/logical_sequence/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../mathematical_induction/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/matrixshapes/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/metaphor_boolean/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../metaphor_understanding/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../minute_mysteries_qa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/misconceptions/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../misconceptions_russian/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/mnist_ascii/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../modified_arithmetic/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../moral_permissibility/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../movie_recommendation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../mult_data_wrangling/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/multiemo/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../natural_instructions/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/navigate/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../nonsense_words_grammar/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/novel_concepts/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/object_counting/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/odd_one_out/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/operators/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../paragraph_segmentation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/parsinlu_qa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../penguins_in_a_table/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/periodic_elements/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/persian_idioms/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/phrase_relatedness/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/physical_intuition/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/physics/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/physics_questions/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../polish_sequence_labeling/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../presuppositions_as_nli/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/qa_wikidata/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/question_selection/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/real_or_fake_text/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/repeat_copy_logic/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/rephrase/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/riddle_sense/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/ruin_names/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../scientific_press_release/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../semantic_parsing_spider/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/sentence_ambiguity/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../similarities_abstraction/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../simp_turing_concept/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../simple_arithmetic_json/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../simple_ethical_questions/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../simple_text_editing/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/snarks/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/social_iqa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/social_support/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../sports_understanding/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/strange_stories/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/strategyqa/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../sufficient_information/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/suicide_risk/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../swahili_english_proverbs/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../symbol_interpretation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/temporal_sequences/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/tense/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/timedial/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/topical_chat/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../understanding_fables/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/undo_permutation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/unit_conversion/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../unit_interpretation/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/what_is_the_tao/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/which_wiki_edit/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../bigbench/dummy/winowhy/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/word_sorting/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes .../dummy/word_unscrambling/1.0.0/dummy_data.zip | Bin 0 -> 187 bytes 167 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/color/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip create mode 100644 datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip diff --git a/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/color/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/color/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 diff --git a/datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 From 58cc74430ad13925f0ccb1dd18ec8c1439161826 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 12:40:58 -0700 Subject: [PATCH 33/91] immutable dict requirement --- additional-tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index aed7b459a9c..f97094e397c 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -13,3 +13,4 @@ promise gin-config babel editdistance +immutabledict From 57a37c604a37a7a71779ed9b5c658e2d421da21d Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 13:04:17 -0700 Subject: [PATCH 34/91] removing yaml flags text --- datasets/bigbench/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 6fb95cd03cd..558858e080c 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -1,5 +1,4 @@ --- -YAML tags: annotations_creators: - crowdsourced - expert-generated From e2416756dd09b30b9de2403e2a6f0dc5a0a3c865 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 13:27:59 -0700 Subject: [PATCH 35/91] added en to languages --- datasets/bigbench/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 558858e080c..9e2e928df74 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -8,7 +8,8 @@ language_creators: - expert-generated - machine-generated - other -languages: [] +languages: +- en-US licenses: - apache-2.0 multilinguality: From 175e805bfb40a6fa04a36831e848e626fda6b704 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 8 Apr 2022 20:58:28 -0700 Subject: [PATCH 36/91] adding dataset_infos.json --- datasets/bigbench/dataset_infos.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 datasets/bigbench/dataset_infos.json diff --git a/datasets/bigbench/dataset_infos.json b/datasets/bigbench/dataset_infos.json new file mode 100644 index 00000000000..7f604a04db9 --- /dev/null +++ b/datasets/bigbench/dataset_infos.json @@ -0,0 +1 @@ +{"abstract_narrative_understanding": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "abstract_narrative_understanding", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 6574843, "num_examples": 3000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5261643, "num_examples": 2400, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1313224, "num_examples": 600, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 13149710, "size_in_bytes": 13149710}, "anachronisms": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "anachronisms", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 48937, "num_examples": 230, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 39209, "num_examples": 184, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9752, "num_examples": 46, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 97898, "size_in_bytes": 97898}, "analogical_similarity": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "analogical_similarity", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1374163, "num_examples": 323, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1101796, "num_examples": 259, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 272391, "num_examples": 64, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2748350, "size_in_bytes": 2748350}, "analytic_entailment": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "analytic_entailment", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 17367, "num_examples": 70, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 13413, "num_examples": 54, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3978, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 34758, "size_in_bytes": 34758}, "arithmetic": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "arithmetic", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3848183, "num_examples": 15023, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 3078715, "num_examples": 12019, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 769493, "num_examples": 3004, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 7696391, "size_in_bytes": 7696391}, "ascii_word_recognition": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "ascii_word_recognition", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 4985315, "num_examples": 5000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 3997801, "num_examples": 4000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 987542, "num_examples": 1000, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 9970658, "size_in_bytes": 9970658}, "authorship_verification": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "authorship_verification", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 14118946, "num_examples": 880, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 11288769, "num_examples": 704, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2830201, "num_examples": 176, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 28237916, "size_in_bytes": 28237916}, "auto_categorization": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "auto_categorization", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 40618, "num_examples": 328, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 33053, "num_examples": 263, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7594, "num_examples": 65, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 81265, "size_in_bytes": 81265}, "auto_debugging": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "auto_debugging", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 5145, "num_examples": 34, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2682, "num_examples": 18, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2491, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 10318, "size_in_bytes": 10318}, "bbq_lite_json": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "bbq_lite_json", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 6898580, "num_examples": 16076, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5515066, "num_examples": 12866, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1383539, "num_examples": 3210, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 13797185, "size_in_bytes": 13797185}, "bridging_anaphora_resolution_barqa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "bridging_anaphora_resolution_barqa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1971124, "num_examples": 648, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1537357, "num_examples": 519, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 433796, "num_examples": 129, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 3942277, "size_in_bytes": 3942277}, "causal_judgment": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "causal_judgment", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 204974, "num_examples": 190, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 165021, "num_examples": 152, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 39977, "num_examples": 38, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 409972, "size_in_bytes": 409972}, "cause_and_effect": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "cause_and_effect", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 49397, "num_examples": 153, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 39691, "num_examples": 123, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9730, "num_examples": 30, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 98818, "size_in_bytes": 98818}, "checkmate_in_one": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "checkmate_in_one", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3140634, "num_examples": 3498, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2516239, "num_examples": 2799, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 624419, "num_examples": 699, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 6281292, "size_in_bytes": 6281292}, "chess_state_tracking": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "chess_state_tracking", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3270710, "num_examples": 6000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2616922, "num_examples": 4800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 653816, "num_examples": 1200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 6541448, "size_in_bytes": 6541448}, "chinese_remainder_theorem": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "chinese_remainder_theorem", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 153313, "num_examples": 500, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 122679, "num_examples": 400, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 30662, "num_examples": 100, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 306654, "size_in_bytes": 306654}, "cifar10_classification": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "cifar10_classification", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 111049748, "num_examples": 20000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 88804772, "num_examples": 16000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 22245000, "num_examples": 4000, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 222099520, "size_in_bytes": 222099520}, "code_line_description": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "code_line_description", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 33733, "num_examples": 60, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 25583, "num_examples": 44, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8174, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 67490, "size_in_bytes": 67490}, "codenames": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "codenames", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 25234, "num_examples": 85, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 20001, "num_examples": 68, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5262, "num_examples": 17, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 50497, "size_in_bytes": 50497}, "color": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "color", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1638787, "num_examples": 4000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1311087, "num_examples": 3200, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 327724, "num_examples": 800, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 3277598, "size_in_bytes": 3277598}, "common_morpheme": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "common_morpheme", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 12444, "num_examples": 50, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 8490, "num_examples": 34, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3978, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 24912, "size_in_bytes": 24912}, "conceptual_combinations": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "conceptual_combinations", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 58948, "num_examples": 103, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 48087, "num_examples": 84, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 10886, "num_examples": 19, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 117921, "size_in_bytes": 117921}, "conlang_translation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "conlang_translation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 215239, "num_examples": 164, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 173069, "num_examples": 132, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 42198, "num_examples": 32, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 430506, "size_in_bytes": 430506}, "contextual_parametric_knowledge_conflicts": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "contextual_parametric_knowledge_conflicts", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 14594175, "num_examples": 17528, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 11671543, "num_examples": 14023, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2922658, "num_examples": 3505, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 29188376, "size_in_bytes": 29188376}, "crash_blossom": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "crash_blossom", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 12242, "num_examples": 38, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 7037, "num_examples": 22, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5229, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 24508, "size_in_bytes": 24508}, "crass_ai": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "crass_ai", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 22922, "num_examples": 44, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 14172, "num_examples": 28, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8774, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 45868, "size_in_bytes": 45868}, "cryobiology_spanish": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "cryobiology_spanish", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 38754, "num_examples": 146, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 31198, "num_examples": 117, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7581, "num_examples": 29, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 77533, "size_in_bytes": 77533}, "cryptonite": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "cryptonite", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2847756, "num_examples": 26157, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2278424, "num_examples": 20926, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 569360, "num_examples": 5231, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 5695540, "size_in_bytes": 5695540}, "cs_algorithms": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "cs_algorithms", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 273274, "num_examples": 1320, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 218868, "num_examples": 1056, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 54430, "num_examples": 264, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 546572, "size_in_bytes": 546572}, "dark_humor_detection": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "dark_humor_detection", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 26610, "num_examples": 80, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 21315, "num_examples": 64, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5319, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 53244, "size_in_bytes": 53244}, "date_understanding": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "date_understanding", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 95249, "num_examples": 369, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 76443, "num_examples": 296, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 18831, "num_examples": 73, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 190523, "size_in_bytes": 190523}, "disambiguation_qa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "disambiguation_qa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 122626, "num_examples": 258, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 98815, "num_examples": 207, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 23835, "num_examples": 51, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 245276, "size_in_bytes": 245276}, "discourse_marker_prediction": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "discourse_marker_prediction", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2091888, "num_examples": 857, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1667020, "num_examples": 686, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 424892, "num_examples": 171, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 4183800, "size_in_bytes": 4183800}, "disfl_qa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "disfl_qa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 7965803, "num_examples": 8000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 6377339, "num_examples": 6400, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1588492, "num_examples": 1600, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 15931634, "size_in_bytes": 15931634}, "dyck_languages": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "dyck_languages", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1238565, "num_examples": 1000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 991204, "num_examples": 800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 247385, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2477154, "size_in_bytes": 2477154}, "elementary_math_qa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "elementary_math_qa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 13471291, "num_examples": 38160, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 10789985, "num_examples": 30531, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2681331, "num_examples": 7629, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 26942607, "size_in_bytes": 26942607}, "emoji_movie": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "emoji_movie", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 33767, "num_examples": 100, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 27071, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6720, "num_examples": 20, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 67558, "size_in_bytes": 67558}, "emojis_emotion_prediction": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "emojis_emotion_prediction", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 48155, "num_examples": 131, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 38601, "num_examples": 105, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9579, "num_examples": 26, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 96335, "size_in_bytes": 96335}, "empirical_judgments": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "empirical_judgments", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 47574, "num_examples": 99, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 38410, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9188, "num_examples": 19, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 95172, "size_in_bytes": 95172}, "english_proverbs": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "english_proverbs", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 22577, "num_examples": 34, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 12103, "num_examples": 18, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 10499, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 45179, "size_in_bytes": 45179}, "english_russian_proverbs": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "english_russian_proverbs", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 59974, "num_examples": 80, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 48115, "num_examples": 64, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 11883, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 119972, "size_in_bytes": 119972}, "entailed_polarity": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "entailed_polarity", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 25501, "num_examples": 148, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 20419, "num_examples": 119, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5107, "num_examples": 29, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 51027, "size_in_bytes": 51027}, "entailed_polarity_hindi": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "entailed_polarity_hindi", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 57129, "num_examples": 138, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 45895, "num_examples": 111, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 11258, "num_examples": 27, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 114282, "size_in_bytes": 114282}, "epistemic_reasoning": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "epistemic_reasoning", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 887932, "num_examples": 2000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 710731, "num_examples": 1600, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 177225, "num_examples": 400, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1775888, "size_in_bytes": 1775888}, "evaluating_information_essentiality": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "evaluating_information_essentiality", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 77564, "num_examples": 68, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 59660, "num_examples": 52, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 17928, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 155152, "size_in_bytes": 155152}, "fact_checker": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "fact_checker", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1340092, "num_examples": 7154, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1072921, "num_examples": 5724, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 267195, "num_examples": 1430, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2680208, "size_in_bytes": 2680208}, "fantasy_reasoning": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "fantasy_reasoning", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 75987, "num_examples": 201, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 61484, "num_examples": 161, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 14527, "num_examples": 40, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 151998, "size_in_bytes": 151998}, "few_shot_nlg": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "few_shot_nlg", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 75985, "num_examples": 153, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 61906, "num_examples": 123, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 14107, "num_examples": 30, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 151998, "size_in_bytes": 151998}, "figure_of_speech_detection": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "figure_of_speech_detection", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 21823, "num_examples": 59, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 16046, "num_examples": 43, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5801, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 43670, "size_in_bytes": 43670}, "formal_fallacies_syllogisms_negation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "formal_fallacies_syllogisms_negation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 8320026, "num_examples": 14200, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 6657263, "num_examples": 11360, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1662787, "num_examples": 2840, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 16640076, "size_in_bytes": 16640076}, "gem": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "gem", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 36067188, "num_examples": 14802, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 28821034, "num_examples": 11845, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7246182, "num_examples": 2957, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 72134404, "size_in_bytes": 72134404}, "gender_inclusive_sentences_german": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "gender_inclusive_sentences_german", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 126934, "num_examples": 200, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 100676, "num_examples": 160, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 26286, "num_examples": 40, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 253896, "size_in_bytes": 253896}, "general_knowledge": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "general_knowledge", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 21928, "num_examples": 70, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 16900, "num_examples": 54, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5052, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 43880, "size_in_bytes": 43880}, "geometric_shapes": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "geometric_shapes", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 180621, "num_examples": 359, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 145030, "num_examples": 288, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 35616, "num_examples": 71, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 361267, "size_in_bytes": 361267}, "goal_step_wikihow": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "goal_step_wikihow", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3571273, "num_examples": 7053, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2856803, "num_examples": 5643, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 714495, "num_examples": 1410, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 7142571, "size_in_bytes": 7142571}, "gre_reading_comprehension": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "gre_reading_comprehension", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 94319, "num_examples": 31, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 44493, "num_examples": 15, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 49850, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 188662, "size_in_bytes": 188662}, "hhh_alignment": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "hhh_alignment", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 273006, "num_examples": 221, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 212580, "num_examples": 179, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 60451, "num_examples": 42, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 546037, "size_in_bytes": 546037}, "hindi_question_answering": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "hindi_question_answering", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 15155809, "num_examples": 6610, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 11984526, "num_examples": 5288, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3171311, "num_examples": 1322, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 30311646, "size_in_bytes": 30311646}, "hindu_knowledge": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "hindu_knowledge", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 44227, "num_examples": 175, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 35505, "num_examples": 140, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8747, "num_examples": 35, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 88479, "size_in_bytes": 88479}, "hinglish_toxicity": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "hinglish_toxicity", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 60712, "num_examples": 200, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 50081, "num_examples": 160, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 10655, "num_examples": 40, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 121448, "size_in_bytes": 121448}, "human_organs_senses": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "human_organs_senses", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 7995, "num_examples": 42, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 4914, "num_examples": 26, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3105, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 16014, "size_in_bytes": 16014}, "hyperbaton": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "hyperbaton", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 9402856, "num_examples": 50000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 7524430, "num_examples": 40000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1878426, "num_examples": 10000, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 18805712, "size_in_bytes": 18805712}, "identify_math_theorems": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "identify_math_theorems", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 104899, "num_examples": 53, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 70343, "num_examples": 37, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 34581, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 209823, "size_in_bytes": 209823}, "identify_odd_metaphor": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "identify_odd_metaphor", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 27658, "num_examples": 47, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 18183, "num_examples": 31, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9499, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 55340, "size_in_bytes": 55340}, "implicatures": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "implicatures", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 91892, "num_examples": 492, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 73589, "num_examples": 394, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 18329, "num_examples": 98, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 183810, "size_in_bytes": 183810}, "implicit_relations": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "implicit_relations", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 80011, "num_examples": 85, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 64592, "num_examples": 68, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 15445, "num_examples": 17, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 160048, "size_in_bytes": 160048}, "intent_recognition": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "intent_recognition", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 323089, "num_examples": 693, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 258444, "num_examples": 555, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 64670, "num_examples": 138, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 646203, "size_in_bytes": 646203}, "international_phonetic_alphabet_nli": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "international_phonetic_alphabet_nli", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 79408, "num_examples": 126, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 63363, "num_examples": 101, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 16070, "num_examples": 25, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 158841, "size_in_bytes": 158841}, "international_phonetic_alphabet_transliterate": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "international_phonetic_alphabet_transliterate", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 276092, "num_examples": 1003, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 220913, "num_examples": 803, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 55207, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 552212, "size_in_bytes": 552212}, "intersect_geometry": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "intersect_geometry", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 212987847, "num_examples": 249999, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 170383378, "num_examples": 200000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 42604469, "num_examples": 49999, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 425975694, "size_in_bytes": 425975694}, "irony_identification": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "irony_identification", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 28240, "num_examples": 99, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 22972, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5292, "num_examples": 19, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 56504, "size_in_bytes": 56504}, "kanji_ascii": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "kanji_ascii", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 367225, "num_examples": 1092, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 294162, "num_examples": 875, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 73089, "num_examples": 217, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 734476, "size_in_bytes": 734476}, "kannada": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "kannada", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 140859, "num_examples": 316, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 112047, "num_examples": 253, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 28836, "num_examples": 63, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 281742, "size_in_bytes": 281742}, "key_value_maps": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "key_value_maps", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 105199, "num_examples": 101, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 84371, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 20852, "num_examples": 21, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 210422, "size_in_bytes": 210422}, "known_unknowns": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "known_unknowns", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 8002, "num_examples": 46, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5166, "num_examples": 30, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2860, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 16028, "size_in_bytes": 16028}, "language_games": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "language_games", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 979913, "num_examples": 2128, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 783352, "num_examples": 1704, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 196589, "num_examples": 424, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1959854, "size_in_bytes": 1959854}, "language_identification": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "language_identification", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 7391247, "num_examples": 10000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5920832, "num_examples": 8000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1470439, "num_examples": 2000, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 14782518, "size_in_bytes": 14782518}, "linguistic_mappings": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "linguistic_mappings", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1327183, "num_examples": 15527, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1061698, "num_examples": 12426, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 265514, "num_examples": 3101, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2654395, "size_in_bytes": 2654395}, "linguistics_puzzles": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "linguistics_puzzles", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1746302, "num_examples": 2000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1398341, "num_examples": 1600, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 347989, "num_examples": 400, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 3492632, "size_in_bytes": 3492632}, "list_functions": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "list_functions", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2679536, "num_examples": 10750, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2162181, "num_examples": 8700, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 517356, "num_examples": 2050, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 5359073, "size_in_bytes": 5359073}, "logic_grid_puzzle": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "logic_grid_puzzle", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1456816, "num_examples": 1000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1160620, "num_examples": 800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 296220, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2913656, "size_in_bytes": 2913656}, "logical_args": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "logical_args", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 43630, "num_examples": 32, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 21108, "num_examples": 16, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 22546, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 87284, "size_in_bytes": 87284}, "logical_deduction": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "logical_deduction", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1057966, "num_examples": 1500, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 842792, "num_examples": 1200, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 215198, "num_examples": 300, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2115956, "size_in_bytes": 2115956}, "logical_fallacy_detection": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "logical_fallacy_detection", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 721360, "num_examples": 2800, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 577159, "num_examples": 2240, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 144225, "num_examples": 560, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1442744, "size_in_bytes": 1442744}, "logical_sequence": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "logical_sequence", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 22771, "num_examples": 39, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 12687, "num_examples": 23, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 10108, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 45566, "size_in_bytes": 45566}, "mathematical_induction": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "mathematical_induction", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 19069, "num_examples": 69, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 15028, "num_examples": 53, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 4065, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 38162, "size_in_bytes": 38162}, "matrixshapes": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "matrixshapes", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1131160, "num_examples": 4462, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 906536, "num_examples": 3570, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 224653, "num_examples": 892, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2262349, "size_in_bytes": 2262349}, "metaphor_boolean": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "metaphor_boolean", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 214127, "num_examples": 680, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 170993, "num_examples": 544, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 43158, "num_examples": 136, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 428278, "size_in_bytes": 428278}, "metaphor_understanding": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "metaphor_understanding", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 201033, "num_examples": 234, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 162243, "num_examples": 188, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 38814, "num_examples": 46, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 402090, "size_in_bytes": 402090}, "minute_mysteries_qa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "minute_mysteries_qa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3245380, "num_examples": 477, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2623861, "num_examples": 383, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 621544, "num_examples": 94, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 6490785, "size_in_bytes": 6490785}, "misconceptions": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "misconceptions", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 45923, "num_examples": 219, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 37336, "num_examples": 176, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8611, "num_examples": 43, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 91870, "size_in_bytes": 91870}, "misconceptions_russian": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "misconceptions_russian", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 17035, "num_examples": 49, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 11008, "num_examples": 33, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6051, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 34094, "size_in_bytes": 34094}, "mnist_ascii": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "mnist_ascii", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 61836204, "num_examples": 69984, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 49497056, "num_examples": 55988, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 12339173, "num_examples": 13996, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 123672433, "size_in_bytes": 123672433}, "modified_arithmetic": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "modified_arithmetic", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1221771, "num_examples": 6000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 977487, "num_examples": 4800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 244312, "num_examples": 1200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2443570, "size_in_bytes": 2443570}, "moral_permissibility": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "moral_permissibility", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 162221, "num_examples": 342, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 128918, "num_examples": 274, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 33328, "num_examples": 68, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 324467, "size_in_bytes": 324467}, "movie_dialog_same_or_different": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "movie_dialog_same_or_different", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 28664867, "num_examples": 50000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 22904157, "num_examples": 40000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5760710, "num_examples": 10000, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 57329734, "size_in_bytes": 57329734}, "movie_recommendation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "movie_recommendation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 173894, "num_examples": 500, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 139210, "num_examples": 400, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 34708, "num_examples": 100, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 347812, "size_in_bytes": 347812}, "mult_data_wrangling": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "mult_data_wrangling", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 626432, "num_examples": 7854, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 508664, "num_examples": 6380, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 117797, "num_examples": 1474, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1252893, "size_in_bytes": 1252893}, "multiemo": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "multiemo", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 651075683, "num_examples": 1437281, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 520893617, "num_examples": 1149873, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 130182066, "num_examples": 287408, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1302151366, "size_in_bytes": 1302151366}, "natural_instructions": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "natural_instructions", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 355963087, "num_examples": 193250, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 284939871, "num_examples": 154615, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 71023216, "num_examples": 38635, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 711926174, "size_in_bytes": 711926174}, "navigate": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "navigate", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 226212, "num_examples": 1000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 181282, "num_examples": 800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 44954, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 452448, "size_in_bytes": 452448}, "nonsense_words_grammar": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "nonsense_words_grammar", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 11164, "num_examples": 50, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 7632, "num_examples": 34, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3556, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 22352, "size_in_bytes": 22352}, "novel_concepts": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "novel_concepts", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 16115, "num_examples": 32, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 8165, "num_examples": 16, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7974, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 32254, "size_in_bytes": 32254}, "object_counting": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "object_counting", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 149708, "num_examples": 1000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 119737, "num_examples": 800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 29999, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 299444, "size_in_bytes": 299444}, "odd_one_out": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "odd_one_out", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 13932, "num_examples": 86, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 11293, "num_examples": 69, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2664, "num_examples": 17, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 27889, "size_in_bytes": 27889}, "operators": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "operators", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 32490, "num_examples": 210, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 25986, "num_examples": 168, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6532, "num_examples": 42, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 65008, "size_in_bytes": 65008}, "paragraph_segmentation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "paragraph_segmentation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 56847660, "num_examples": 9000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 45675248, "num_examples": 7200, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 11172440, "num_examples": 1800, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 113695348, "size_in_bytes": 113695348}, "parsinlu_qa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "parsinlu_qa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 456870, "num_examples": 1050, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 367126, "num_examples": 840, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 89768, "num_examples": 210, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 913764, "size_in_bytes": 913764}, "parsinlu_reading_comprehension": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "parsinlu_reading_comprehension", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 573891, "num_examples": 518, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 455908, "num_examples": 415, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 118011, "num_examples": 103, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1147810, "size_in_bytes": 1147810}, "penguins_in_a_table": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "penguins_in_a_table", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 76121, "num_examples": 149, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 61435, "num_examples": 120, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 14711, "num_examples": 29, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 152267, "size_in_bytes": 152267}, "periodic_elements": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "periodic_elements", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 287051, "num_examples": 654, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 230973, "num_examples": 524, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 56104, "num_examples": 130, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 574128, "size_in_bytes": 574128}, "persian_idioms": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "persian_idioms", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 28658, "num_examples": 66, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 21740, "num_examples": 50, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6942, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 57340, "size_in_bytes": 57340}, "phrase_relatedness": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "phrase_relatedness", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 30277, "num_examples": 100, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 23847, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6454, "num_examples": 20, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 60578, "size_in_bytes": 60578}, "physical_intuition": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "physical_intuition", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 23810, "num_examples": 81, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 19373, "num_examples": 65, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 4461, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 47644, "size_in_bytes": 47644}, "physics": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "physics", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 120407, "num_examples": 229, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 96261, "num_examples": 184, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 24170, "num_examples": 45, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 240838, "size_in_bytes": 240838}, "physics_questions": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "physics_questions", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 18407, "num_examples": 54, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 13435, "num_examples": 38, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5000, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 36842, "size_in_bytes": 36842}, "play_dialog_same_or_different": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "play_dialog_same_or_different", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 3143716, "num_examples": 3264, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2517056, "num_examples": 2612, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 626685, "num_examples": 652, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 6287457, "size_in_bytes": 6287457}, "polish_sequence_labeling": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "polish_sequence_labeling", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 18082770, "num_examples": 12812, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 14472058, "num_examples": 10250, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3610741, "num_examples": 2562, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 36165569, "size_in_bytes": 36165569}, "presuppositions_as_nli": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "presuppositions_as_nli", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 502914, "num_examples": 735, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 401080, "num_examples": 588, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 101860, "num_examples": 147, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1005854, "size_in_bytes": 1005854}, "qa_wikidata": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "qa_wikidata", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1421667, "num_examples": 20321, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1137007, "num_examples": 16257, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 284660, "num_examples": 4064, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2843334, "size_in_bytes": 2843334}, "question_selection": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "question_selection", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2487986, "num_examples": 1582, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1990739, "num_examples": 1266, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 497272, "num_examples": 316, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 4975997, "size_in_bytes": 4975997}, "real_or_fake_text": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "real_or_fake_text", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 53684101, "num_examples": 15088, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 42896484, "num_examples": 12072, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 10787642, "num_examples": 3016, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 107368227, "size_in_bytes": 107368227}, "reasoning_about_colored_objects": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "reasoning_about_colored_objects", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 912440, "num_examples": 2000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 733608, "num_examples": 1600, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 178857, "num_examples": 400, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1824905, "size_in_bytes": 1824905}, "repeat_copy_logic": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "repeat_copy_logic", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 6710, "num_examples": 32, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 3357, "num_examples": 16, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3381, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 13448, "size_in_bytes": 13448}, "rephrase": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "rephrase", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 34260, "num_examples": 78, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 27396, "num_examples": 62, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6892, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 68548, "size_in_bytes": 68548}, "riddle_sense": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "riddle_sense", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 15569, "num_examples": 49, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 10791, "num_examples": 33, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 4802, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 31162, "size_in_bytes": 31162}, "ruin_names": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "ruin_names", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 144391, "num_examples": 448, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 115420, "num_examples": 359, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 28997, "num_examples": 89, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 288808, "size_in_bytes": 288808}, "salient_translation_error_detection": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "salient_translation_error_detection", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1142524, "num_examples": 998, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 913543, "num_examples": 799, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 229006, "num_examples": 199, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2285073, "size_in_bytes": 2285073}, "scientific_press_release": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "scientific_press_release", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 13725, "num_examples": 50, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 9287, "num_examples": 34, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 4466, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 27478, "size_in_bytes": 27478}, "semantic_parsing_in_context_sparc": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "semantic_parsing_in_context_sparc", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1525025, "num_examples": 1155, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 1248535, "num_examples": 924, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 276518, "num_examples": 231, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 3050078, "size_in_bytes": 3050078}, "semantic_parsing_spider": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "semantic_parsing_spider", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1265902, "num_examples": 1034, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 973996, "num_examples": 828, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 291934, "num_examples": 206, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2531832, "size_in_bytes": 2531832}, "sentence_ambiguity": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "sentence_ambiguity", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 8215, "num_examples": 60, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 6017, "num_examples": 44, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 2222, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 16454, "size_in_bytes": 16454}, "similarities_abstraction": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "similarities_abstraction", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 23490, "num_examples": 76, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 18609, "num_examples": 60, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 4906, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 47005, "size_in_bytes": 47005}, "simp_turing_concept": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simp_turing_concept", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1018473, "num_examples": 6390, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 813887, "num_examples": 5112, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 204614, "num_examples": 1278, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2036974, "size_in_bytes": 2036974}, "simple_arithmetic_json": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_arithmetic_json", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1177, "num_examples": 30, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 570, "num_examples": 14, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 635, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2382, "size_in_bytes": 2382}, "simple_arithmetic_json_multiple_choice": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_arithmetic_json_multiple_choice", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 953, "num_examples": 8, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 0, "num_examples": 0, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 0, "num_examples": 0, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 953, "size_in_bytes": 953}, "simple_arithmetic_json_subtasks": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_arithmetic_json_subtasks", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1177, "num_examples": 30, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 601, "num_examples": 15, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 604, "num_examples": 15, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2382, "size_in_bytes": 2382}, "simple_arithmetic_multiple_targets_json": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_arithmetic_multiple_targets_json", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 444, "num_examples": 10, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 0, "num_examples": 0, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 0, "num_examples": 0, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 444, "size_in_bytes": 444}, "simple_ethical_questions": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_ethical_questions", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 76615, "num_examples": 115, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 60357, "num_examples": 92, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 16282, "num_examples": 23, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 153254, "size_in_bytes": 153254}, "simple_text_editing": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "simple_text_editing", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 27899, "num_examples": 47, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 18501, "num_examples": 31, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 9426, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 55826, "size_in_bytes": 55826}, "snarks": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "snarks", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 45810, "num_examples": 181, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 37069, "num_examples": 145, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8766, "num_examples": 36, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 91645, "size_in_bytes": 91645}, "social_iqa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "social_iqa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 644154, "num_examples": 1935, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 516485, "num_examples": 1548, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 127694, "num_examples": 387, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1288333, "size_in_bytes": 1288333}, "social_support": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "social_support", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 367179, "num_examples": 897, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 295177, "num_examples": 718, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 72027, "num_examples": 179, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 734383, "size_in_bytes": 734383}, "sports_understanding": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "sports_understanding", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 227049, "num_examples": 986, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 181649, "num_examples": 789, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 45425, "num_examples": 197, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 454123, "size_in_bytes": 454123}, "strange_stories": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "strange_stories", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 120620, "num_examples": 174, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 98157, "num_examples": 140, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 22489, "num_examples": 34, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 241266, "size_in_bytes": 241266}, "strategyqa": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "strategyqa", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 660851, "num_examples": 2289, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 528381, "num_examples": 1832, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 132494, "num_examples": 457, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1321726, "size_in_bytes": 1321726}, "sufficient_information": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "sufficient_information", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 9458, "num_examples": 39, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5625, "num_examples": 23, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 3861, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 18944, "size_in_bytes": 18944}, "suicide_risk": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "suicide_risk", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 38001, "num_examples": 40, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 23106, "num_examples": 24, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 14919, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 76026, "size_in_bytes": 76026}, "swahili_english_proverbs": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "swahili_english_proverbs", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 90367, "num_examples": 153, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 72569, "num_examples": 123, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 17822, "num_examples": 30, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 180758, "size_in_bytes": 180758}, "swedish_to_german_proverbs": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "swedish_to_german_proverbs", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 35273, "num_examples": 72, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 27325, "num_examples": 56, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7972, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 70570, "size_in_bytes": 70570}, "symbol_interpretation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "symbol_interpretation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1149725, "num_examples": 990, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 927947, "num_examples": 795, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 221803, "num_examples": 195, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2299475, "size_in_bytes": 2299475}, "temporal_sequences": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "temporal_sequences", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 687735, "num_examples": 1000, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 550332, "num_examples": 800, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 137427, "num_examples": 200, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1375494, "size_in_bytes": 1375494}, "tense": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "tense", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 43946, "num_examples": 286, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 35523, "num_examples": 229, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 8452, "num_examples": 57, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 87921, "size_in_bytes": 87921}, "timedial": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "timedial", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2764478, "num_examples": 2550, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2218234, "num_examples": 2040, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 546268, "num_examples": 510, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 5528980, "size_in_bytes": 5528980}, "topical_chat": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "topical_chat", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 30930629, "num_examples": 22295, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 24829540, "num_examples": 17836, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6101090, "num_examples": 4459, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 61861259, "size_in_bytes": 61861259}, "tracking_shuffled_objects": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "tracking_shuffled_objects", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 2779059, "num_examples": 3750, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 2226511, "num_examples": 3000, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 552572, "num_examples": 750, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 5558142, "size_in_bytes": 5558142}, "understanding_fables": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "understanding_fables", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 227915, "num_examples": 189, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 181138, "num_examples": 152, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 46801, "num_examples": 37, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 455854, "size_in_bytes": 455854}, "undo_permutation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "undo_permutation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 196443, "num_examples": 300, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 158827, "num_examples": 240, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 37641, "num_examples": 60, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 392911, "size_in_bytes": 392911}, "unit_conversion": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "unit_conversion", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 4040317, "num_examples": 23936, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 3239699, "num_examples": 19151, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 800619, "num_examples": 4785, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 8080635, "size_in_bytes": 8080635}, "unit_interpretation": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "unit_interpretation", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 37463, "num_examples": 100, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 30023, "num_examples": 80, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 7464, "num_examples": 20, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 74950, "size_in_bytes": 74950}, "unnatural_in_context_learning": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "unnatural_in_context_learning", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 4609162, "num_examples": 73420, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 3687332, "num_examples": 58736, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 921830, "num_examples": 14684, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 9218324, "size_in_bytes": 9218324}, "vitaminc_fact_verification": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "vitaminc_fact_verification", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 32389297, "num_examples": 54668, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 25911838, "num_examples": 43735, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 6477483, "num_examples": 10933, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 64778618, "size_in_bytes": 64778618}, "what_is_the_tao": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "what_is_the_tao", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 13306, "num_examples": 36, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 7467, "num_examples": 20, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 5863, "num_examples": 16, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 26636, "size_in_bytes": 26636}, "which_wiki_edit": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "which_wiki_edit", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 6332065, "num_examples": 571, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 5234181, "num_examples": 457, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 1097909, "num_examples": 114, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 12664155, "size_in_bytes": 12664155}, "winowhy": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "winowhy", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 1003532, "num_examples": 2862, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 801404, "num_examples": 2290, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 202153, "num_examples": 572, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 2007089, "size_in_bytes": 2007089}, "word_sorting": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "word_sorting", "version": {"version_str": "0.0.0", "description": null, "major": 0, "minor": 0, "patch": 0}, "splits": {"default": {"name": "default", "num_bytes": 491320, "num_examples": 1900, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 392956, "num_examples": 1520, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 98392, "num_examples": 380, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 982668, "size_in_bytes": 982668}, "word_unscrambling": {"description": "The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to\nprobe large language models, and extrapolate their future capabilities.\n", "citation": "@InProceedings{bigbench,\ntitle = {Beyond the Imitation Game: Quantifying and extrapolating the\ncapabilities of language models},\nauthor={BIG-Bench Collaboration\n},\nyear={2022}\n}\n", "homepage": "https://github.com/google/BIG-bench", "license": "Apache License 2.0", "features": {"idx": {"dtype": "int32", "id": null, "_type": "Value"}, "inputs": {"dtype": "string", "id": null, "_type": "Value"}, "targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_targets": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "multiple_choice_scores": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "bigbench", "config_name": "word_unscrambling", "version": "0.0.0", "splits": {"default": {"name": "default", "num_bytes": 883507, "num_examples": 8917, "dataset_name": "bigbench"}, "train": {"name": "train", "num_bytes": 706675, "num_examples": 7134, "dataset_name": "bigbench"}, "validation": {"name": "validation", "num_bytes": 176860, "num_examples": 1783, "dataset_name": "bigbench"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1767042, "size_in_bytes": 1767042}} \ No newline at end of file From d68d469a1d8146fb4306be2579ecc0ef5dcb9268 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Mon, 11 Apr 2022 11:45:32 -0400 Subject: [PATCH 37/91] Update datasets/bigbench/README.md Co-authored-by: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> --- datasets/bigbench/README.md | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 9e2e928df74..42e2c426509 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -99,29 +99,15 @@ To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo Each dataset contains 5 features. For example an instance from the `emoji_movie` task is: -- `idx`: - - 0 -- `inputs`: - - Q: What movie does this emoji describe? 👦👓⚡️ - choice: harry potter - choice: shutter island - choice: inglourious basterds - choice: die hard - choice: moonlight - A: - -- `targets`: - - ['harry potter'] -- `multiple_choice_targets`: - - ['harry potter', 'shutter island', 'die hard', 'inglourious basterds', 'moonlight'] - -- `multiple_choice_scores`: - - [1, 0, 0, 0, 0] +``` +{ + "idx": 0, + "inputs": "Q: What movie does this emoji describe? 👦👓⚡️\n choice: harry potter\n. choice: shutter island\n. choice: inglourious basterds\n. choice: die hard\n. choice: moonlight\nA:" + "targets": ["harry potter"], + "multiple_choice_targets":["harry potter", "shutter island", "die hard", "inglourious basterds", "moonlight"], + "multiple_choice_scores": [1, 0, 0, 0, 0] +} +``` For tasks that do not have multiple choice targets, the lists are empty. From 51184c1172f7f84969e3694b69f187480ca26206 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Mon, 11 Apr 2022 11:45:43 -0400 Subject: [PATCH 38/91] Update datasets/bigbench/README.md Co-authored-by: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> --- datasets/bigbench/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 42e2c426509..a6dc7b481d2 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -84,7 +84,7 @@ The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark BIG-Bench consists of both json and programmatic tasks. This implementation in HuggingFace datasets implements - - 24 BIG-bench Lite tasks + - 24 BIG-bench Lite tasks - 167 BIG-bench json tasks (includes BIG-bench Lite) To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo](https://github.com/google/BIG-bench) From 1ebb159992a24af7fcd80f16f966771510c9f839 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 08:57:29 -0700 Subject: [PATCH 39/91] adding bigbench to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9d8c1cc0243..b2276319e22 100644 --- a/setup.py +++ b/setup.py @@ -143,6 +143,7 @@ "py7zr", "tldextract", "zstandard", + "git+https://github.com/google/BIG-bench.git", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From f50c546bfcd908a57493c57a0f275a5bd3742137 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 08:58:29 -0700 Subject: [PATCH 40/91] removing dependencies from additional-test-requirements.txt as they will now be in setup.py --- additional-tests-requirements.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index f97094e397c..a827c308c9f 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -2,15 +2,3 @@ unbabel-comet>=1.0.0;python_version>'3.6' git+https://github.com/google-research/bleurt.git git+https://github.com/ns-moosavi/coval.git git+https://github.com/hendrycks/math.git -git+https://github.com/google/BIG-bench.git -t5 -seqio -tensorflow_datasets -tensorflow_metadata -tensorflow_text -tensorflow_hub -promise -gin-config -babel -editdistance -immutabledict From e0b186fa478b6dedd17e5f5d90f5f26fb8a17a3f Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 08:59:22 -0700 Subject: [PATCH 41/91] cleaning up bigbench import --- datasets/bigbench/bigbench.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index b5a1bc9ee86..6f0a3fa86dc 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -24,18 +24,9 @@ logger = datasets.logging.get_logger(__name__) -try: - import bigbench.api.util as bb_utils - import bigbench.bbseqio.bigbench_bridge as bbb - from bigbench.bbseqio import bigbench_json_paths as bb_json_paths - -except ModuleNotFoundError as error: - logger.info( - "Failed to import bigbench. Please see https://github.com/google/BIG-bench or install" - "using 'pip install git+https://github.com/google/BIG-bench.git'" - ) - logger.error(error) - raise +import bigbench.api.util as bb_utils # From: git+https://github.com/google/BIG-bench.git +import bigbench.bbseqio.bigbench_bridge as bbb +from bigbench.bbseqio import bigbench_json_paths as bb_json_paths _CITATION = """\ @InProceedings{bigbench, From 630eee6a2224172a7551429f982b0b648eb21adf Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 09:08:28 -0700 Subject: [PATCH 42/91] delete dummy data --- .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/anachronisms/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../analogical_similarity/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../analytic_entailment/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/arithmetic/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../ascii_word_recognition/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../authorship_verification/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../auto_categorization/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/auto_debugging/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/bbq_lite_json/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/causal_judgment/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/cause_and_effect/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/checkmate_in_one/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../chess_state_tracking/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../cifar10_classification/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../code_line_description/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/codenames/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/color/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/common_morpheme/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../conceptual_combinations/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../conlang_translation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/crash_blossom/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/crass_ai/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../cryobiology_spanish/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/cryptonite/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/cs_algorithms/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dark_humor_detection/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/date_understanding/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/disambiguation_qa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/dyck_languages/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/elementary_math_qa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/emoji_movie/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../empirical_judgments/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/english_proverbs/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../english_russian_proverbs/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/entailed_polarity/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../entailed_polarity_hindi/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../epistemic_reasoning/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/fact_checker/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/fantasy_reasoning/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/few_shot_nlg/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/general_knowledge/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/geometric_shapes/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/goal_step_wikihow/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/hhh_alignment/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../hindi_question_answering/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/hindu_knowledge/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/hinglish_toxicity/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../human_organs_senses/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/hyperbaton/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../identify_math_theorems/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../identify_odd_metaphor/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/implicatures/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/implicit_relations/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/intent_recognition/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/intersect_geometry/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../irony_identification/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/kanji_ascii/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/kannada/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/key_value_maps/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/known_unknowns/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/language_games/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../language_identification/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../linguistic_mappings/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../linguistics_puzzles/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/list_functions/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/logic_grid_puzzle/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/logical_args/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/logical_deduction/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/logical_sequence/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../mathematical_induction/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/matrixshapes/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/metaphor_boolean/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../metaphor_understanding/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../minute_mysteries_qa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/misconceptions/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../misconceptions_russian/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/mnist_ascii/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../modified_arithmetic/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../moral_permissibility/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../movie_recommendation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../mult_data_wrangling/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/multiemo/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../natural_instructions/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/navigate/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../nonsense_words_grammar/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/novel_concepts/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/object_counting/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/odd_one_out/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/operators/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../paragraph_segmentation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/parsinlu_qa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../penguins_in_a_table/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/periodic_elements/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/persian_idioms/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/phrase_relatedness/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/physical_intuition/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/physics/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/physics_questions/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../polish_sequence_labeling/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../presuppositions_as_nli/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/qa_wikidata/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/question_selection/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/real_or_fake_text/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/repeat_copy_logic/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/rephrase/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/riddle_sense/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/ruin_names/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../scientific_press_release/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../semantic_parsing_spider/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/sentence_ambiguity/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../similarities_abstraction/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../simp_turing_concept/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../simple_arithmetic_json/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../simple_ethical_questions/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../simple_text_editing/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/snarks/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/social_iqa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/social_support/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../sports_understanding/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/strange_stories/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/strategyqa/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../sufficient_information/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/suicide_risk/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../swahili_english_proverbs/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../symbol_interpretation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/temporal_sequences/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/tense/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/timedial/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/topical_chat/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../understanding_fables/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/undo_permutation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/unit_conversion/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../unit_interpretation/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/what_is_the_tao/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/which_wiki_edit/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../bigbench/dummy/winowhy/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/word_sorting/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes .../dummy/word_unscrambling/1.0.0/dummy_data.zip | Bin 187 -> 0 bytes 167 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/color/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip delete mode 100644 datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip diff --git a/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/anachronisms/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/analogical_similarity/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/analytic_entailment/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/arithmetic/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/ascii_word_recognition/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/authorship_verification/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/auto_categorization/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/auto_debugging/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/bbq_lite_json/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/bridging_anaphora_resolution_barqa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/causal_judgment/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cause_and_effect/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/checkmate_in_one/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/chess_state_tracking/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/chinese_remainder_theorem/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cifar10_classification/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/code_line_description/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/codenames/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/color/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/color/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/common_morpheme/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/conceptual_combinations/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/conlang_translation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/contextual_parametric_knowledge_conflicts/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/crash_blossom/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/crass_ai/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cryobiology_spanish/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cryptonite/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/cs_algorithms/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/dark_humor_detection/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/date_understanding/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/disambiguation_qa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/discourse_marker_prediction/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/disfl_qa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/dyck_languages/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/elementary_math_qa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/emoji_movie/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/emojis_emotion_prediction/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/empirical_judgments/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/english_proverbs/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/english_russian_proverbs/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/entailed_polarity/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/entailed_polarity_hindi/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/epistemic_reasoning/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/evaluating_information_essentiality/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/fact_checker/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/fantasy_reasoning/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/few_shot_nlg/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/figure_of_speech_detection/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/formal_fallacies_syllogisms_negation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gem/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gender_inclusive_sentences_german/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/general_knowledge/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/geometric_shapes/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/goal_step_wikihow/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/gre_reading_comprehension/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hhh_alignment/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hindi_question_answering/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hindu_knowledge/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hinglish_toxicity/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/human_organs_senses/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/hyperbaton/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/identify_math_theorems/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/identify_odd_metaphor/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/implicatures/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/implicit_relations/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/intent_recognition/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/international_phonetic_alphabet_nli/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/international_phonetic_alphabet_transliterate/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/intersect_geometry/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/irony_identification/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/kanji_ascii/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/kannada/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/key_value_maps/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/known_unknowns/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/language_games/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/language_identification/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/linguistic_mappings/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/linguistics_puzzles/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/list_functions/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logic_grid_puzzle/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_args/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_deduction/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_fallacy_detection/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/logical_sequence/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mathematical_induction/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/matrixshapes/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/metaphor_boolean/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/metaphor_understanding/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/minute_mysteries_qa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/misconceptions/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/misconceptions_russian/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mnist_ascii/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/modified_arithmetic/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/moral_permissibility/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/movie_dialog_same_or_different/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/movie_recommendation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/mult_data_wrangling/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/multiemo/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/natural_instructions/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/navigate/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/nonsense_words_grammar/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/novel_concepts/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/object_counting/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/odd_one_out/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/operators/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/paragraph_segmentation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/parsinlu_qa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/parsinlu_reading_comprehension/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/penguins_in_a_table/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/periodic_elements/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/persian_idioms/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/phrase_relatedness/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physical_intuition/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physics/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/physics_questions/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/play_dialog_same_or_different/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/polish_sequence_labeling/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/presuppositions_as_nli/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/qa_wikidata/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/question_selection/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/real_or_fake_text/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/reasoning_about_colored_objects/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/repeat_copy_logic/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/rephrase/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/riddle_sense/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/ruin_names/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/salient_translation_error_detection/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/scientific_press_release/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/semantic_parsing_in_context_sparc/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/semantic_parsing_spider/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sentence_ambiguity/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/similarities_abstraction/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simp_turing_concept/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json_multiple_choice/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_json_subtasks/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_arithmetic_multiple_targets_json/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_ethical_questions/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/simple_text_editing/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/snarks/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/social_iqa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/social_support/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sports_understanding/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/strange_stories/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/strategyqa/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/sufficient_information/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/suicide_risk/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/swahili_english_proverbs/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/swedish_to_german_proverbs/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/symbol_interpretation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/temporal_sequences/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/tense/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/timedial/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/topical_chat/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/tracking_shuffled_objects/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/understanding_fables/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/undo_permutation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unit_conversion/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unit_interpretation/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/unnatural_in_context_learning/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/vitaminc_fact_verification/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/what_is_the_tao/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/which_wiki_edit/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/winowhy/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/word_sorting/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB diff --git a/datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/word_unscrambling/1.0.0/dummy_data.zip deleted file mode 100644 index 474d48dce7121142afc70d07b545281cf0411189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB From 8c54f80c2131a34c9e5c1f1de04d6df266c2a9f6 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 09:09:15 -0700 Subject: [PATCH 43/91] adding back in abstract_narrative_understanding dummy data --- .../1.0.0/dummy_data.zip | Bin 0 -> 187 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip diff --git a/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..474d48dce7121142afc70d07b545281cf0411189 GIT binary patch literal 187 zcmWIWW@Zs#U|`^2_?*-ca`@!8eXc-W9uRW@aY|`!Ze@H*VoBoauv0!O12zR2TnG#} zbw*o5SCg%1%9MsZla4I<62fA>-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB literal 0 HcmV?d00001 From 2386aad35e8c4bb18ff927ded877a3445766ccbb Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 12:44:50 -0700 Subject: [PATCH 44/91] updating setup dependencies to temp branch that imports bigbench without datasets=1.17. PR in bigbench on the way --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2276319e22..b0d368d0dc9 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "git+https://github.com/google/BIG-bench.git", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From e7e72f1180cd44a12b934ebaeb64738c75115930 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 12:55:55 -0700 Subject: [PATCH 45/91] fixing isort format issue --- datasets/bigbench/bigbench.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 6f0a3fa86dc..67a6666b4bc 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -28,6 +28,7 @@ import bigbench.bbseqio.bigbench_bridge as bbb from bigbench.bbseqio import bigbench_json_paths as bb_json_paths + _CITATION = """\ @InProceedings{bigbench, title = {Beyond the Imitation Game: Quantifying and extrapolating the From 1a764b99ffa06867bd4a230b1fd9a889a7b5d9ec Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 13:00:52 -0700 Subject: [PATCH 46/91] more formatting for isort and flake8 --- datasets/bigbench/bigbench.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 67a6666b4bc..18d0b1e9e4d 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -19,15 +19,14 @@ from typing import Optional -import datasets - - -logger = datasets.logging.get_logger(__name__) - import bigbench.api.util as bb_utils # From: git+https://github.com/google/BIG-bench.git import bigbench.bbseqio.bigbench_bridge as bbb from bigbench.bbseqio import bigbench_json_paths as bb_json_paths +import datasets + + +logger = datasets.logging.get_logger(__name__) _CITATION = """\ @InProceedings{bigbench, From cb98a40905841bf1c73629c4f44695b6dc84f8a8 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 11 Apr 2022 15:54:10 -0700 Subject: [PATCH 47/91] commenting out bigbench to test apache-beam issue --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b0d368d0dc9..eb8fc58bd1b 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", + #"BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 128597b647a5dc3ef5995d102c7a8f431c96fff4 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 12 Apr 2022 08:21:12 -0700 Subject: [PATCH 48/91] adding bigbench back into setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb8fc58bd1b..b0d368d0dc9 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - #"BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From ae1e201aa2d224f51553eac4a60b24512dbbb6ce Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Wed, 13 Apr 2022 13:35:15 -0400 Subject: [PATCH 49/91] updating language section in readme --- datasets/bigbench/README.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index a6dc7b481d2..1a74c619979 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -84,14 +84,30 @@ The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark BIG-Bench consists of both json and programmatic tasks. This implementation in HuggingFace datasets implements - - 24 BIG-bench Lite tasks - - 167 BIG-bench json tasks (includes BIG-bench Lite) + + - 24 BIG-bench Lite tasks + + - 167 BIG-bench json tasks (includes BIG-bench Lite) -To study the remaining programmatic tasks, please see the BIG-bench GitHub [repo](https://github.com/google/BIG-bench) +To study the remaining programmatic tasks, please see the [BIG-bench GitHub repo](https://github.com/google/BIG-bench) ### Languages -[More Information Needed] +Althoug predomenantly English, BIG-bench contains a tasks in over 1000 written languages, as well as some synthetic and programming languages. +See [BIG-bench organized by keywords](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md). Relevant keywords include `multilingual`, `non-english`, `low-resource-language`, `translation`. + +For tasks specifically targeting low-resource languages, see the table below: + +Task Name | Languages | +--|--| +Conlang Translation Problems | English, German, Finnish, Abma, Apinayé, Inapuri, Ndebele, Palauan| +Kannada Riddles | Kannada| +Language Identification | 1000 languages | +Swahili English Proverbs | Swahili | +Which Wiki Edit | English, Russian, Spanish, German, French, Turkish, Japanese, Vietnamese, Chinese, Arabic, Norwegian, Tagalog| + + + ## Dataset Structure From 675c3692fcba46c11ac9d473bdf2a374e84993f0 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Wed, 13 Apr 2022 13:41:46 -0400 Subject: [PATCH 50/91] Updated Dataset Creation in readme --- datasets/bigbench/README.md | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 1a74c619979..80b0a1e2e6d 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -145,33 +145,11 @@ For standard evaluation on BIG-bench, we recommend using the `default` split, an ## Dataset Creation -### Curation Rationale +BIG-bench tasks were collaboratively submitted through GitHub pull requests. -[More Information Needed] - -### Source Data - -#### Initial Data Collection and Normalization - -Tasks were contributed by the research community through [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench), and [reviewed](https://github.com/google/BIG-bench/blob/main/docs/doc.md#submission-review-process) by members of the collaboration. - -#### Who are the source language producers? - -[More Information Needed] - -### Annotations +Each task went through a review and meta-review process with criteria outlined in the [BIG-bench repository documentation](https://github.com/google/BIG-bench/blob/main/docs/doc.md#submission-review-process). +Each task was required to describe the data source and curation methods on the task README page. -#### Annotation process - -[More Information Needed] - -#### Who are the annotators? - -[More Information Needed] - -### Personal and Sensitive Information - -[More Information Needed] ## Considerations for Using the Data From 75f57f3e5d98c41cb498ec07d6216ba9b40ca9a7 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Wed, 13 Apr 2022 14:01:51 -0400 Subject: [PATCH 51/91] More updates to README --- datasets/bigbench/README.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 80b0a1e2e6d..bd70c2abac3 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -153,32 +153,26 @@ Each task was required to describe the data source and curation methods on the t ## Considerations for Using the Data -### Social Impact of Dataset +BIG-bench contains a wide range of tasks, some of which are sensitive and should be used with care. -[More Information Needed] +Some tasks are specifically designed to test biases and failures common to large language models, and so may elicit inappropriate or harmful responses. +For a more thourough discussion see the [BIG-bench paper](in progress). -### Discussion of Biases +To view tasks designed to probe pro-social behavior, including alignment, social, racial, gender, religious or political bias; toxicity; inclusion; and other issues please see tasks under the [pro-social behavior keywords](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md#pro-social-behavior) on the BIG-bench repository. -[More Information Needed] - -### Other Known Limitations - -[More Information Needed] ## Additional Information -### Dataset Curators - -[More Information Needed] +For a more thorough discussion of all aspects of BIG-bench including dataset creation and evaluations see the BIG-bench repository [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench) and paper [] ### Licensing Information [Apache License 2.0](https://github.com/google/BIG-bench/blob/main/LICENSE) ### Citation Information - -A paper is in progress. Until then, please cite the BIG-bench collaboration and the GitHub [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench). +TODO(andersjohanandreassen): Add bibtex citation block. ### Contributions +For a full list of contributors to the BIG-bench dataset, see the paper []. -Thanks to [@andersjohanandreassen](https://github.com/andersjohanandreassen) for adding this dataset. +Thanks to [@andersjohanandreassen](https://github.com/andersjohanandreassen) and [@ethansdyer](https://github.com/ethansdyer) for adding this dataset to HuggingFace. From f55deddba377c0770181b413575d55999db8397a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen <43357549+andersjohanandreassen@users.noreply.github.com> Date: Wed, 13 Apr 2022 14:04:12 -0400 Subject: [PATCH 52/91] updated index in readme --- datasets/bigbench/README.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index bd70c2abac3..1931552ec8d 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -48,25 +48,17 @@ task_ids: - [Table of Contents](#table-of-contents) - [Dataset Description](#dataset-description) - [Dataset Summary](#dataset-summary) - - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards) + - [Supported Tasks](#supported-tasks) - [Languages](#languages) - [Dataset Structure](#dataset-structure) - [Data Instances](#data-instances) - [Data Fields](#data-fields) - [Data Splits](#data-splits) - [Dataset Creation](#dataset-creation) - - [Curation Rationale](#curation-rationale) - - [Source Data](#source-data) - - [Annotations](#annotations) - - [Personal and Sensitive Information](#personal-and-sensitive-information) - [Considerations for Using the Data](#considerations-for-using-the-data) - - [Social Impact of Dataset](#social-impact-of-dataset) - - [Discussion of Biases](#discussion-of-biases) - - [Other Known Limitations](#other-known-limitations) - [Additional Information](#additional-information) - - [Dataset Curators](#dataset-curators) - [Licensing Information](#licensing-information) - - [Citation Inform ation](#citation-information) + - [Citation Information](#citation-information) - [Contributions](#contributions) ## Dataset Description @@ -80,7 +72,7 @@ task_ids: The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models and extrapolate their future capabilities. Tasks included in BIG-bench are summarized by keyword [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md), and by task name [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/README.md). A paper introducing the benchmark, including evaluation results on large language models, is currently in preparation. -### Supported Tasks and Leaderboards +### Supported Tasks BIG-Bench consists of both json and programmatic tasks. This implementation in HuggingFace datasets implements From 201589657db2233f48cc7d0b5081f77c7c9109bf Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 15 Apr 2022 06:08:19 -0700 Subject: [PATCH 53/91] update path to bigbench install --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b0d368d0dc9..d6ce592eefc 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@remove_datasets_dependency", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 3969599be2204f39b07f51842bda473ff47a287b Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 15 Apr 2022 10:40:33 -0700 Subject: [PATCH 54/91] dummy update to trigger tests --- datasets/bigbench/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 1931552ec8d..4e184b61457 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -80,7 +80,7 @@ This implementation in HuggingFace datasets implements - 24 BIG-bench Lite tasks - 167 BIG-bench json tasks (includes BIG-bench Lite) - + To study the remaining programmatic tasks, please see the [BIG-bench GitHub repo](https://github.com/google/BIG-bench) ### Languages From 987ae39ec2af2bc1c89dcbe0605487b89d4a91c8 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 07:11:13 -0700 Subject: [PATCH 55/91] switching to experimental bigbench branch to resolve dependency issues --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47a4b08ad5e..3ee08259c3d 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 8b3107a9ae414cc926557a5ff5fb72ac4390e6e6 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 07:24:02 -0700 Subject: [PATCH 56/91] switching to experimental bigbench branch to resolve dependency issues --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ee08259c3d..60ef66050f5 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 3f5f8eb573842d7b388dacb0bbe8b987d75a27d4 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 07:40:23 -0700 Subject: [PATCH 57/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60ef66050f5..3ee08259c3d 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 4cce1ba9d86f94deff1341eefd2ca3d2b5790acd Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 07:55:25 -0700 Subject: [PATCH 58/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ee08259c3d..60ef66050f5 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From e6ae146da42a1e0ee4eafa6d37bc00c2b12b5484 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 10:55:33 -0700 Subject: [PATCH 59/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60ef66050f5..3ee08259c3d 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 4b7ff887c2a3fa95b22c80e09b6011da718c323b Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 12:51:06 -0700 Subject: [PATCH 60/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ee08259c3d..60ef66050f5 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 1ad556f6ab847f1be0c3a1b4e63bf34808caefc3 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 19 Apr 2022 20:37:56 -0700 Subject: [PATCH 61/91] add version --- datasets/bigbench/bigbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 18d0b1e9e4d..9368fd2a5ef 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -115,7 +115,7 @@ class Bigbench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - BUILDER_CONFIGS = [BigBenchConfig(name=name) for name in bb_utils.get_all_json_task_names()] + BUILDER_CONFIGS = [BigBenchConfig(name=name, version=VERSION) for name in bb_utils.get_all_json_task_names()] def _info(self): features = datasets.Features( From d6d6810d6748ae74fe811c3c6e3017a42cd7f034 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Wed, 20 Apr 2022 10:31:59 -0700 Subject: [PATCH 62/91] switching back to tar install for bigbenhc --- datasets/bigbench/bigbench.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 9368fd2a5ef..db31fdf8c15 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -115,7 +115,8 @@ class Bigbench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - BUILDER_CONFIGS = [BigBenchConfig(name=name, version=VERSION) for name in bb_utils.get_all_json_task_names()] + BUILDER_CONFIGS = [BigBenchConfig(name=name, version=datasets.Version("1.0.0")) + for name in bb_utils.get_all_json_task_names()] def _info(self): features = datasets.Features( diff --git a/setup.py b/setup.py index 60ef66050f5..d42c2417031 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench.git@private_dependency_checker", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 70e81ebeee046350165dae87d29e45614395fd9b Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Wed, 20 Apr 2022 10:34:34 -0700 Subject: [PATCH 63/91] formatting --- datasets/bigbench/bigbench.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index db31fdf8c15..f508da731fd 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -115,8 +115,9 @@ class Bigbench(datasets.GeneratorBasedBuilder): BUILDER_CONFIG_CLASS = BigBenchConfig - BUILDER_CONFIGS = [BigBenchConfig(name=name, version=datasets.Version("1.0.0")) - for name in bb_utils.get_all_json_task_names()] + BUILDER_CONFIGS = [ + BigBenchConfig(name=name, version=datasets.Version("1.0.0")) for name in bb_utils.get_all_json_task_names() + ] def _info(self): features = datasets.Features( From 86f4c2233b30045701050c15dd40fd8040ecab6a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 21 Apr 2022 06:05:20 -0700 Subject: [PATCH 64/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d42c2417031..47a4b08ad5e 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From ef33a84b1daa862e84addeff748fa698a717ab54 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 21 Apr 2022 06:20:09 -0700 Subject: [PATCH 65/91] trigger test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47a4b08ad5e..d42c2417031 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 650b4c282cea6dc76d616086f696bba2cd3b9fc3 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 21 Apr 2022 07:24:46 -0700 Subject: [PATCH 66/91] Adding back in empty sections to readme --- datasets/bigbench/README.md | 51 +++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 4e184b61457..ce53749e3e6 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -48,15 +48,23 @@ task_ids: - [Table of Contents](#table-of-contents) - [Dataset Description](#dataset-description) - [Dataset Summary](#dataset-summary) - - [Supported Tasks](#supported-tasks) + - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards) - [Languages](#languages) - [Dataset Structure](#dataset-structure) - [Data Instances](#data-instances) - [Data Fields](#data-fields) - [Data Splits](#data-splits) - [Dataset Creation](#dataset-creation) + - [Curation Rationale](#curation-rationale) + - [Source Data](#source-data) + - [Annotations](#annotations) + - [Personal and Sensitive Information](#personal-and-sensitive-information) - [Considerations for Using the Data](#considerations-for-using-the-data) + - [Social Impact of Dataset](#social-impact-of-dataset) + - [Discussion of Biases](#discussion-of-biases) + - [Other Known Limitations](#other-known-limitations) - [Additional Information](#additional-information) + - [Dataset Curators](#dataset-curators) - [Licensing Information](#licensing-information) - [Citation Information](#citation-information) - [Contributions](#contributions) @@ -68,11 +76,12 @@ task_ids: - **Leaderboard:** - **Point of Contact:** [bigbench@googlegroups.com](mailto:bigbench@googlegroups.com) + ### Dataset Summary The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models and extrapolate their future capabilities. Tasks included in BIG-bench are summarized by keyword [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md), and by task name [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/README.md). A paper introducing the benchmark, including evaluation results on large language models, is currently in preparation. -### Supported Tasks +### Supported Tasks and Leaderboards BIG-Bench consists of both json and programmatic tasks. This implementation in HuggingFace datasets implements @@ -142,6 +151,24 @@ BIG-bench tasks were collaboratively submitted through GitHub pull requests. Each task went through a review and meta-review process with criteria outlined in the [BIG-bench repository documentation](https://github.com/google/BIG-bench/blob/main/docs/doc.md#submission-review-process). Each task was required to describe the data source and curation methods on the task README page. +### Curation Rationale + +[More Information Needed] + +### Source Data + +[More Information Needed] + + +### Annotations + +[More Information Needed] + + +### Personal and Sensitive Information + +[More Information Needed] + ## Considerations for Using the Data @@ -153,10 +180,30 @@ For a more thourough discussion see the [BIG-bench paper](in progress). To view tasks designed to probe pro-social behavior, including alignment, social, racial, gender, religious or political bias; toxicity; inclusion; and other issues please see tasks under the [pro-social behavior keywords](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md#pro-social-behavior) on the BIG-bench repository. +### Social Impact of Dataset + +[More Information Needed] + + +### Discussion of Biases + +[More Information Needed] + + +### Other Known Limitations + +[More Information Needed] + + ## Additional Information For a more thorough discussion of all aspects of BIG-bench including dataset creation and evaluations see the BIG-bench repository [https://github.com/google/BIG-bench](https://github.com/google/BIG-bench) and paper [] +### Dataset Curators + +[More Information Needed] + + ### Licensing Information [Apache License 2.0](https://github.com/google/BIG-bench/blob/main/LICENSE) From 7fd8348ca8bd194a289a46398d3b4c35c10afbe8 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 21 Apr 2022 07:43:28 -0700 Subject: [PATCH 67/91] fix sections of readme --- datasets/bigbench/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index ce53749e3e6..10021e05dba 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -157,11 +157,23 @@ Each task was required to describe the data source and curation methods on the t ### Source Data +#### Initial Data Collection and Normalization + +[More Information Needed] + +#### Who are the source language producers? + [More Information Needed] ### Annotations +#### Annotation process + +[More Information Needed] + +#### Who are the annotators? + [More Information Needed] From 3eaf284534c5de0484609a2705b3215f3b26e247 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 26 Apr 2022 08:10:50 -0700 Subject: [PATCH 68/91] Modifying dummy data. Passing empty folder to test failure --- .../1.0.0/dummy_data.zip | Bin 187 -> 120 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip index 474d48dce7121142afc70d07b545281cf0411189..fd2c21b3c749cb2e5e37bcd9c4fe7daec2be5554 100644 GIT binary patch literal 120 zcmWIWW@Zs#0D-DJ7SW(9RqJ4Gj8jEHiE&DMi7OtJiwck4J5({gcd+L7sO!z0I~}+U;qFB From 6f2da0c2742640e917ad7f41a8f3b369527cb3b4 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 26 Apr 2022 08:42:06 -0700 Subject: [PATCH 69/91] deleting dummy data --- .../1.0.0/dummy_data.zip | Bin 120 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip diff --git a/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip b/datasets/bigbench/dummy/abstract_narrative_understanding/1.0.0/dummy_data.zip deleted file mode 100644 index fd2c21b3c749cb2e5e37bcd9c4fe7daec2be5554..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 120 zcmWIWW@Zs#0D Date: Thu, 5 May 2022 16:49:58 +0200 Subject: [PATCH 70/91] pin s3 dependencies to speed up pip backtracking --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d42c2417031..e06bd342d54 100644 --- a/setup.py +++ b/setup.py @@ -118,9 +118,9 @@ # optional dependencies "apache-beam>=2.26.0", "elasticsearch<8.0.0", # 8.0 asks users to provide hosts or cloud_id when instantiating ElastictSearch() - "aiobotocore", - "boto3", - "botocore", + "aiobotocore==1.4.2", # to be compatible with boto3==1.17.106 - both have strong dependencies on botocore + "boto3==1.17.106", # to be compatible with aiobotocore==1.4.2 - both have strong dependencies on botocore + "botocore==1.20.106", # to be compatible with aiobotocore and boto3 "faiss-cpu>=1.6.4", "fsspec[s3]", "moto[s3,server]==2.0.4", From af41167048aba99ec2c6e8506c42df0ecf9e6984 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Thu, 5 May 2022 19:12:46 +0200 Subject: [PATCH 71/91] disable s3 test on windows for now --- tests/test_arrow_dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_arrow_dataset.py b/tests/test_arrow_dataset.py index 30e662e8018..a96b1c9e4a7 100644 --- a/tests/test_arrow_dataset.py +++ b/tests/test_arrow_dataset.py @@ -3053,6 +3053,10 @@ def test_pickle_dataset_after_transforming_the_table(in_memory, method_and_param assert dataset._data.table == reloaded_dataset._data.table +@pytest.mark.skipif( + os.name == "nt", + reason='On Windows CircleCI, it raises botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "http://127.0.0.1:5555/test"', +) # TODO(QL): find what's wrong with CircleCI @require_s3 def test_dummy_dataset_serialize_s3(s3, dataset): mock_bucket = s3_test_bucket_name From 88aeb20abf5c3c75572ff1cb18238fc6b253b565 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 6 May 2022 15:31:25 +0200 Subject: [PATCH 72/91] disable other test too --- tests/test_arrow_dataset.py | 2 +- tests/test_dataset_dict.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_arrow_dataset.py b/tests/test_arrow_dataset.py index a96b1c9e4a7..4c6da70bcce 100644 --- a/tests/test_arrow_dataset.py +++ b/tests/test_arrow_dataset.py @@ -3054,7 +3054,7 @@ def test_pickle_dataset_after_transforming_the_table(in_memory, method_and_param @pytest.mark.skipif( - os.name == "nt", + os.name == "nt" and os.getenv("CIRCLECI") == "true", reason='On Windows CircleCI, it raises botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "http://127.0.0.1:5555/test"', ) # TODO(QL): find what's wrong with CircleCI @require_s3 diff --git a/tests/test_dataset_dict.py b/tests/test_dataset_dict.py index cb799e69398..814d04ee370 100644 --- a/tests/test_dataset_dict.py +++ b/tests/test_dataset_dict.py @@ -655,6 +655,10 @@ def test_datasetdict_from_text_split(split, text_path, tmp_path): assert all(dataset[split].split == split for split in path.keys()) +@pytest.mark.skipif( + os.name == "nt" and os.getenv("CIRCLECI") == "true", + reason='On Windows CircleCI, it raises botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "http://127.0.0.1:5555/test"', +) # TODO(QL): find what's wrong with CircleCI @require_s3 def test_dummy_dataset_serialize_s3(s3, dataset): dsets = DatasetDict({"train": dataset, "test": dataset.select(range(2))}) From 62ff3d9e9d6088e9107515a01aad96a22573b281 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 6 May 2022 16:00:10 +0200 Subject: [PATCH 73/91] explicitely require sentencepiece --- datasets/bigbench/bigbench.py | 3 ++- setup.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index f508da731fd..98a2c2489b4 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -19,8 +19,9 @@ from typing import Optional -import bigbench.api.util as bb_utils # From: git+https://github.com/google/BIG-bench.git +import bigbench.api.util as bb_utils # From: "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz" import bigbench.bbseqio.bigbench_bridge as bbb +from sentencepiece import sentencepiece_model_pb2 # noqa: this is also required by bigbench.api.util from bigbench.bbseqio import bigbench_json_paths as bb_json_paths import datasets diff --git a/setup.py b/setup.py index e06bd342d54..d3122ca27e4 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,8 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 0441a0ffe92952b80cdf2489e9ef250e4ac55569 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> Date: Fri, 6 May 2022 16:05:29 +0200 Subject: [PATCH 74/91] Apply suggestions from code review --- datasets/bigbench/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 10021e05dba..6e0b5904cc1 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -9,7 +9,7 @@ language_creators: - machine-generated - other languages: -- en-US +- en licenses: - apache-2.0 multilinguality: @@ -221,9 +221,10 @@ For a more thorough discussion of all aspects of BIG-bench including dataset cre [Apache License 2.0](https://github.com/google/BIG-bench/blob/main/LICENSE) ### Citation Information -TODO(andersjohanandreassen): Add bibtex citation block. + +To be added soon ! ### Contributions -For a full list of contributors to the BIG-bench dataset, see the paper []. +For a full list of contributors to the BIG-bench dataset, see the paper. Thanks to [@andersjohanandreassen](https://github.com/andersjohanandreassen) and [@ethansdyer](https://github.com/ethansdyer) for adding this dataset to HuggingFace. From 890a49f52db6c1fe376cd0f3f537d5a8dd4d8430 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 6 May 2022 16:09:23 +0200 Subject: [PATCH 75/91] style --- datasets/bigbench/bigbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 98a2c2489b4..a575f013e51 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -21,8 +21,8 @@ import bigbench.api.util as bb_utils # From: "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz" import bigbench.bbseqio.bigbench_bridge as bbb -from sentencepiece import sentencepiece_model_pb2 # noqa: this is also required by bigbench.api.util from bigbench.bbseqio import bigbench_json_paths as bb_json_paths +from sentencepiece import sentencepiece_model_pb2 # noqa: this is also required by bigbench.api.util import datasets From 1317e8f02b8f38e2b939323283f904f616c6e55e Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Mon, 16 May 2022 07:15:34 -0700 Subject: [PATCH 76/91] Minor updates to README --- datasets/bigbench/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index 6e0b5904cc1..cabea625520 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -94,7 +94,7 @@ To study the remaining programmatic tasks, please see the [BIG-bench GitHub repo ### Languages -Althoug predomenantly English, BIG-bench contains a tasks in over 1000 written languages, as well as some synthetic and programming languages. +Although predominantly English, BIG-bench contains tasks in over 1000 written languages, as well as some synthetic and programming languages. See [BIG-bench organized by keywords](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md). Relevant keywords include `multilingual`, `non-english`, `low-resource-language`, `translation`. For tasks specifically targeting low-resource languages, see the table below: @@ -187,7 +187,7 @@ Each task was required to describe the data source and curation methods on the t BIG-bench contains a wide range of tasks, some of which are sensitive and should be used with care. Some tasks are specifically designed to test biases and failures common to large language models, and so may elicit inappropriate or harmful responses. -For a more thourough discussion see the [BIG-bench paper](in progress). +For a more thorough discussion see the [BIG-bench paper](in progress). To view tasks designed to probe pro-social behavior, including alignment, social, racial, gender, religious or political bias; toxicity; inclusion; and other issues please see tasks under the [pro-social behavior keywords](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md#pro-social-behavior) on the BIG-bench repository. From e356fb66f8255f1752743c2bd5e3ae26a3167515 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Tue, 17 May 2022 17:11:41 -0700 Subject: [PATCH 77/91] trigger test --- datasets/bigbench/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/README.md b/datasets/bigbench/README.md index cabea625520..a86ba85daa9 100644 --- a/datasets/bigbench/README.md +++ b/datasets/bigbench/README.md @@ -99,7 +99,7 @@ See [BIG-bench organized by keywords](https://github.com/google/BIG-bench/blob/m For tasks specifically targeting low-resource languages, see the table below: -Task Name | Languages | +Task Name | Languages | --|--| Conlang Translation Problems | English, German, Finnish, Abma, Apinayé, Inapuri, Ndebele, Palauan| Kannada Riddles | Kannada| From 54894e0d35529308c0afdf6cdc7bd38016cc63c9 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 15:18:52 -0700 Subject: [PATCH 78/91] temporarily commenting out 4/5 test to focus on windows issue. --- .circleci/config.yml | 132 +++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4219fab8c7e..69fed65604e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,37 +4,37 @@ orbs: win: circleci/windows@2.2.0 jobs: - run_dataset_script_tests_pyarrow_latest: - working_directory: ~/datasets - docker: - - image: cimg/python:3.6 - resource_class: medium - steps: - - checkout - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev - - run: pip install --upgrade pip - - run: python -m venv venv - - run: source venv/bin/activate - - run: pip install .[tests] - - run: pip install -r additional-tests-requirements.txt --no-deps - - run: pip install pyarrow --upgrade - - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ +# run_dataset_script_tests_pyarrow_latest: +# working_directory: ~/datasets +# docker: +# - image: cimg/python:3.6 +# resource_class: medium +# steps: +# - checkout +# - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev +# - run: pip install --upgrade pip +# - run: python -m venv venv +# - run: source venv/bin/activate +# - run: pip install .[tests] +# - run: pip install -r additional-tests-requirements.txt --no-deps +# - run: pip install pyarrow --upgrade +# - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ - run_dataset_script_tests_pyarrow_5: - working_directory: ~/datasets - docker: - - image: cimg/python:3.6 - resource_class: medium - steps: - - checkout - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev - - run: pip install --upgrade pip - - run: python -m venv venv - - run: source venv/bin/activate - - run: pip install .[tests] - - run: pip install -r additional-tests-requirements.txt --no-deps - - run: pip install pyarrow==5.0.0 - - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ +# run_dataset_script_tests_pyarrow_5: +# working_directory: ~/datasets +# docker: +# - image: cimg/python:3.6 +# resource_class: medium +# steps: +# - checkout +# - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev +# - run: pip install --upgrade pip +# - run: python -m venv venv +# - run: source venv/bin/activate +# - run: pip install .[tests] +# - run: pip install -r additional-tests-requirements.txt --no-deps +# - run: pip install pyarrow==5.0.0 +# - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ run_dataset_script_tests_pyarrow_latest_WIN: working_directory: ~/datasets @@ -57,46 +57,46 @@ jobs: $env:HF_SCRIPTS_VERSION="master" python -m pytest -n 2 --dist loadfile -sv ./tests/ - run_dataset_script_tests_pyarrow_5_WIN: - working_directory: ~/datasets - executor: - name: win/default - shell: powershell - steps: - - checkout - - run: | - conda init powershell - conda update conda - conda create -n py37 python=3.7 pytorch --yes - - run: | - conda activate py37 - pip install .[tests] - pip install -r additional-tests-requirements.txt --no-deps - pip install pyarrow==5.0.0 - - run: | - conda activate py37 - $env:HF_SCRIPTS_VERSION="master" - python -m pytest -n 2 --dist loadfile -sv ./tests/ +# run_dataset_script_tests_pyarrow_5_WIN: +# working_directory: ~/datasets +# executor: +# name: win/default +# shell: powershell +# steps: +# - checkout +# - run: | +# conda init powershell +# conda update conda +# conda create -n py37 python=3.7 pytorch --yes +# - run: | +# conda activate py37 +# pip install .[tests] +# pip install -r additional-tests-requirements.txt --no-deps +# pip install pyarrow==5.0.0 +# - run: | +# conda activate py37 +# $env:HF_SCRIPTS_VERSION="master" +# python -m pytest -n 2 --dist loadfile -sv ./tests/ - check_code_quality: - working_directory: ~/datasets - docker: - - image: circleci/python:3.6 - resource_class: medium - parallelism: 1 - steps: - - checkout - - run: sudo pip install .[quality] - - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics - - run: isort --check-only tests src benchmarks datasets metrics - - run: flake8 tests src benchmarks datasets metrics +# check_code_quality: +# working_directory: ~/datasets +# docker: +# - image: circleci/python:3.6 +# resource_class: medium +# parallelism: 1 +# steps: +# - checkout +# - run: sudo pip install .[quality] +# - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics +# - run: isort --check-only tests src benchmarks datasets metrics +# - run: flake8 tests src benchmarks datasets metrics workflows: version: 2 build_and_test: jobs: - - check_code_quality - - run_dataset_script_tests_pyarrow_latest - - run_dataset_script_tests_pyarrow_5 + # - check_code_quality + # - run_dataset_script_tests_pyarrow_latest + # - run_dataset_script_tests_pyarrow_5 - run_dataset_script_tests_pyarrow_latest_WIN - - run_dataset_script_tests_pyarrow_5_WIN + # - run_dataset_script_tests_pyarrow_5_WIN From bd72c7ff50b022bb5cc2675dd3c2548f40eb7874 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 15:19:29 -0700 Subject: [PATCH 79/91] switching to private github repo for bigbench while debugging current errors --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3122ca27e4..a5c181bc9dc 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,8 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + #"bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@private_dependency_checker", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece # metrics dependencies "bert_score>=0.3.6", From 9bb5eb87adef2adddefca68433ce9b8c2ee1b17a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 15:33:43 -0700 Subject: [PATCH 80/91] long filenames workaround attempt --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69fed65604e..47dec2e68ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,6 +47,7 @@ jobs: conda init powershell conda update conda conda create -n py37 python=3.7 pytorch --yes + git config --system core.longpaths true - run: | conda activate py37 pip install .[tests] From 1cc4d9c8ee2e886aeaa226e40f1bdfacd9426764 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 15:54:18 -0700 Subject: [PATCH 81/91] trigger new tests after adding editdistance on bigbench requirements remotely --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a5c181bc9dc..9a1d531cdd0 100644 --- a/setup.py +++ b/setup.py @@ -144,7 +144,7 @@ "tldextract", "zstandard", #"bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@private_dependency_checker", + "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@private_dependency_checker", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece # metrics dependencies "bert_score>=0.3.6", From 1f85c7a85f4a06e83e10ea8b81df7f3409bb11b6 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 21:36:38 -0700 Subject: [PATCH 82/91] adding sacremoses --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9a1d531cdd0..183404ebf5d 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,7 @@ #"bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@private_dependency_checker", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece + "sacremoses", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 7828cd2bcb59d8d23979ac4d95e427c4b2672a9a Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 22:12:24 -0700 Subject: [PATCH 83/91] changing to bigbench tar file --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 183404ebf5d..b6cd2b6b913 100644 --- a/setup.py +++ b/setup.py @@ -143,8 +143,7 @@ "py7zr", "tldextract", "zstandard", - #"bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", - "BIG-bench @ git+https://github.com/andersjohanandreassen/BIG-bench@private_dependency_checker", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece "sacremoses", # metrics dependencies From db51cd2df079f15e1e43cf55b184ac3b7655a915 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 19 May 2022 22:27:41 -0700 Subject: [PATCH 84/91] reverting circleci to run all test --- .circleci/config.yml | 132 +++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47dec2e68ab..24406b20168 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,37 +4,37 @@ orbs: win: circleci/windows@2.2.0 jobs: -# run_dataset_script_tests_pyarrow_latest: -# working_directory: ~/datasets -# docker: -# - image: cimg/python:3.6 -# resource_class: medium -# steps: -# - checkout -# - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev -# - run: pip install --upgrade pip -# - run: python -m venv venv -# - run: source venv/bin/activate -# - run: pip install .[tests] -# - run: pip install -r additional-tests-requirements.txt --no-deps -# - run: pip install pyarrow --upgrade -# - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ + run_dataset_script_tests_pyarrow_latest: + working_directory: ~/datasets + docker: + - image: cimg/python:3.6 + resource_class: medium + steps: + - checkout + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: python -m venv venv + - run: source venv/bin/activate + - run: pip install .[tests] + - run: pip install -r additional-tests-requirements.txt --no-deps + - run: pip install pyarrow --upgrade + - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ -# run_dataset_script_tests_pyarrow_5: -# working_directory: ~/datasets -# docker: -# - image: cimg/python:3.6 -# resource_class: medium -# steps: -# - checkout -# - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev -# - run: pip install --upgrade pip -# - run: python -m venv venv -# - run: source venv/bin/activate -# - run: pip install .[tests] -# - run: pip install -r additional-tests-requirements.txt --no-deps -# - run: pip install pyarrow==5.0.0 -# - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ + run_dataset_script_tests_pyarrow_5: + working_directory: ~/datasets + docker: + - image: cimg/python:3.6 + resource_class: medium + steps: + - checkout + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: python -m venv venv + - run: source venv/bin/activate + - run: pip install .[tests] + - run: pip install -r additional-tests-requirements.txt --no-deps + - run: pip install pyarrow==5.0.0 + - run: HF_SCRIPTS_VERSION=master HF_ALLOW_CODE_EVAL=1 python -m pytest -d --tx 2*popen//python=python3.6 --dist loadfile -sv ./tests/ run_dataset_script_tests_pyarrow_latest_WIN: working_directory: ~/datasets @@ -58,46 +58,46 @@ jobs: $env:HF_SCRIPTS_VERSION="master" python -m pytest -n 2 --dist loadfile -sv ./tests/ -# run_dataset_script_tests_pyarrow_5_WIN: -# working_directory: ~/datasets -# executor: -# name: win/default -# shell: powershell -# steps: -# - checkout -# - run: | -# conda init powershell -# conda update conda -# conda create -n py37 python=3.7 pytorch --yes -# - run: | -# conda activate py37 -# pip install .[tests] -# pip install -r additional-tests-requirements.txt --no-deps -# pip install pyarrow==5.0.0 -# - run: | -# conda activate py37 -# $env:HF_SCRIPTS_VERSION="master" -# python -m pytest -n 2 --dist loadfile -sv ./tests/ + run_dataset_script_tests_pyarrow_5_WIN: + working_directory: ~/datasets + executor: + name: win/default + shell: powershell + steps: + - checkout + - run: | + conda init powershell + conda update conda + conda create -n py37 python=3.7 pytorch --yes + - run: | + conda activate py37 + pip install .[tests] + pip install -r additional-tests-requirements.txt --no-deps + pip install pyarrow==5.0.0 + - run: | + conda activate py37 + $env:HF_SCRIPTS_VERSION="master" + python -m pytest -n 2 --dist loadfile -sv ./tests/ -# check_code_quality: -# working_directory: ~/datasets -# docker: -# - image: circleci/python:3.6 -# resource_class: medium -# parallelism: 1 -# steps: -# - checkout -# - run: sudo pip install .[quality] -# - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics -# - run: isort --check-only tests src benchmarks datasets metrics -# - run: flake8 tests src benchmarks datasets metrics + check_code_quality: + working_directory: ~/datasets + docker: + - image: circleci/python:3.6 + resource_class: medium + parallelism: 1 + steps: + - checkout + - run: sudo pip install .[quality] + - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics + - run: isort --check-only tests src benchmarks datasets metrics + - run: flake8 tests src benchmarks datasets metrics workflows: version: 2 build_and_test: jobs: - # - check_code_quality - # - run_dataset_script_tests_pyarrow_latest - # - run_dataset_script_tests_pyarrow_5 + - check_code_quality + - run_dataset_script_tests_pyarrow_latest + - run_dataset_script_tests_pyarrow_5 - run_dataset_script_tests_pyarrow_latest_WIN - # - run_dataset_script_tests_pyarrow_5_WIN + - run_dataset_script_tests_pyarrow_5_WIN From 6bb370771510a3354e5539a69069e225c5cb0677 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 20 May 2022 07:14:55 -0700 Subject: [PATCH 85/91] Experiment: pinning apache-beam and removing sacremoses --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b6cd2b6b913..750b82b1c62 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ "pytest-datadir", "pytest-xdist", # optional dependencies - "apache-beam>=2.26.0", + "apache-beam==2.39.0rc2", "elasticsearch<8.0.0", # 8.0 asks users to provide hosts or cloud_id when instantiating ElastictSearch() "aiobotocore==1.4.2", # to be compatible with boto3==1.17.106 - both have strong dependencies on botocore "boto3==1.17.106", # to be compatible with aiobotocore==1.4.2 - both have strong dependencies on botocore @@ -145,7 +145,7 @@ "zstandard", "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece - "sacremoses", + #"sacremoses", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 40fcd54c99f0072ca6f4b6cdedf5d814fec6e77c Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 20 May 2022 07:46:26 -0700 Subject: [PATCH 86/91] reverting circleci edit --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24406b20168..4219fab8c7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,6 @@ jobs: conda init powershell conda update conda conda create -n py37 python=3.7 pytorch --yes - git config --system core.longpaths true - run: | conda activate py37 pip install .[tests] From 0e621cf759ba43783d50178cabe68c8199b2274f Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 20 May 2022 07:47:05 -0700 Subject: [PATCH 87/91] adding sacremoses back in and reverting apache-beam version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 750b82b1c62..b6cd2b6b913 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ "pytest-datadir", "pytest-xdist", # optional dependencies - "apache-beam==2.39.0rc2", + "apache-beam>=2.26.0", "elasticsearch<8.0.0", # 8.0 asks users to provide hosts or cloud_id when instantiating ElastictSearch() "aiobotocore==1.4.2", # to be compatible with boto3==1.17.106 - both have strong dependencies on botocore "boto3==1.17.106", # to be compatible with aiobotocore==1.4.2 - both have strong dependencies on botocore @@ -145,7 +145,7 @@ "zstandard", "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece - #"sacremoses", + "sacremoses", # metrics dependencies "bert_score>=0.3.6", "rouge_score", From 9c251176019502857c4167a8aef4b2915f87e3fb Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Thu, 26 May 2022 21:13:10 -0700 Subject: [PATCH 88/91] someone changed dependencies of bigbench. testing to see if it causes issues --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b6cd2b6b913..86c7c88baf7 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece "sacremoses", # metrics dependencies From 363d421118eddca32ac7bf3f7b6b2c01b15e0ec1 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Fri, 27 May 2022 16:07:36 -0700 Subject: [PATCH 89/91] updated bigbench requirements. triggering tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86c7c88baf7..b6cd2b6b913 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ "py7zr", "tldextract", "zstandard", - "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", + "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz", "sentencepiece", # bigbench requires t5 which requires seqio which requires sentencepiece "sacremoses", # metrics dependencies From 1b706ed90a8121fd888c26aaae5cb8c2ed901584 Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Wed, 1 Jun 2022 12:25:58 -0700 Subject: [PATCH 90/91] Update format_fn after breaking change from bigbench --- datasets/bigbench/bigbench.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index a575f013e51..165b544a840 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -19,6 +19,7 @@ from typing import Optional +from bigbench.api import json_task import bigbench.api.util as bb_utils # From: "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz" import bigbench.bbseqio.bigbench_bridge as bbb from bigbench.bbseqio import bigbench_json_paths as bb_json_paths @@ -210,6 +211,7 @@ def _generate_examples( max_examples=self.config.max_examples, json_util=json_util, min_validation_examples=MIN_VALIDATION_EXAMPLES, + format_fn=json_task.default_format_fn, ) ds_list = [ds_fn(split)] elif self.config.subtask_name is not None: @@ -222,6 +224,7 @@ def _generate_examples( max_examples=self.config.max_examples, json_util=json_util, min_validation_examples=min_validation_examples_per_subtask, + format_fn=json_task.default_format_fn, ) ds_list = [ds_fn(split)] else: @@ -239,6 +242,7 @@ def _generate_examples( max_examples=div_or_none(self.config.max_examples, num_subtasks), json_util=json_util, min_validation_examples=min_validation_examples_per_subtask, + format_fn=json_task.default_format_fn, ) ds_list.append(ds_fn(split)) except ValueError as value_error: From a4e03a87f5f256ba0913c42aff6bb9250317d80b Mon Sep 17 00:00:00 2001 From: Anders Johan Andreassen Date: Wed, 1 Jun 2022 23:05:47 -0700 Subject: [PATCH 91/91] code formatting --- datasets/bigbench/bigbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/bigbench/bigbench.py b/datasets/bigbench/bigbench.py index 165b544a840..28079ca1210 100644 --- a/datasets/bigbench/bigbench.py +++ b/datasets/bigbench/bigbench.py @@ -19,9 +19,9 @@ from typing import Optional -from bigbench.api import json_task import bigbench.api.util as bb_utils # From: "bigbench @ https://storage.googleapis.com/public_research_data/bigbench/bigbench-0.0.1.tar.gz" import bigbench.bbseqio.bigbench_bridge as bbb +from bigbench.api import json_task from bigbench.bbseqio import bigbench_json_paths as bb_json_paths from sentencepiece import sentencepiece_model_pb2 # noqa: this is also required by bigbench.api.util