diff --git a/datasets/nlu_evaluation_data/README.md b/datasets/nlu_evaluation_data/README.md new file mode 100644 index 00000000000..9d94e067f77 --- /dev/null +++ b/datasets/nlu_evaluation_data/README.md @@ -0,0 +1,263 @@ +--- +annotations_creators: +- expert-generated +extended: +- original +language_creators: +- expert-generated +languages: +- en +licenses: +- cc-by-4.0 +multilinguality: +- monolingual +size_categories: +- 10K To build the NLU component we collected real user data via Amazon Mechanical Turk (AMT). We designed tasks where the Turker’s goal was to answer questions about how people would interact with the home robot, in a wide range of scenarios designed in advance, namely: alarm, audio, audiobook, calendar, cooking, datetime, email, game, general, IoT, lists, music, news, podcasts, general Q&A, radio, recommendations, social, food takeaway, transport, and weather. +The questions put to Turkers were designed to capture the different requests within each given scenario. +In the ‘calendar’ scenario, for example, these pre-designed intents were included: ‘set event’, ‘delete event’ and ‘query event’. +An example question for intent ‘set event’ is: “How would you ask your PDA to schedule a meeting with someone?” for which a user’s answer example was “Schedule a chat with Adam on Thursday afternoon”. +The Turkers would then type in their answers to these questions and select possible entities from the pre-designed suggested entities list for each of their answers.The Turkers didn’t always follow the instructions fully, e.g. for the specified ‘delete event’ Intent, an answer was: “PDA what is my next event?”; which clearly belongs to ‘query event’ Intent. +We have manually corrected all such errors either during post-processing or the subsequent annotations. + +#### Who are the annotators? + +[More Information Needed] + +### Personal and Sensitive Information + +[More Information Needed] + +## Considerations for Using the Data + +### Social Impact of Dataset + +The purpose of this dataset it to help develop better intent detection systems. + +### Discussion of Biases + +[More Information Needed] + +### Other Known Limitations + +[More Information Needed] + +## Additional Information + +### Dataset Curators + +[More Information Needed] + +### Licensing Information + +Creative Commons Attribution 4.0 International License (CC BY 4.0) + +### Citation Information +``` +@InProceedings{XLiu.etal:IWSDS2019, + author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser}, + title = {Benchmarking Natural Language Understanding Services for building Conversational Agents}, + booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)}, + month = {April}, + year = {2019}, + address = {Ortigia, Siracusa (SR), Italy}, + publisher = {Springer}, + pages = {xxx--xxx}, + url = {http://www.xx.xx/xx/} +} +``` +### Contributions + +Thanks to [@dkajtoch](https://github.com/dkajtoch) for adding this dataset. diff --git a/datasets/nlu_evaluation_data/dataset_infos.json b/datasets/nlu_evaluation_data/dataset_infos.json new file mode 100644 index 00000000000..8d3364b59a5 --- /dev/null +++ b/datasets/nlu_evaluation_data/dataset_infos.json @@ -0,0 +1 @@ +{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples (original dataset has 25716 examples) from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_data", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1447941, "num_examples": 25715, "dataset_name": "nlu_evaluation_data"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1447941, "size_in_bytes": 7315380}} \ No newline at end of file diff --git a/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip b/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip new file mode 100644 index 00000000000..2e72bebc733 Binary files /dev/null and b/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip differ diff --git a/datasets/nlu_evaluation_data/nlu_evaluation_data.py b/datasets/nlu_evaluation_data/nlu_evaluation_data.py new file mode 100644 index 00000000000..07b1c1c0767 --- /dev/null +++ b/datasets/nlu_evaluation_data/nlu_evaluation_data.py @@ -0,0 +1,211 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NLU Evaluation Data.""" + +from __future__ import absolute_import, division, print_function + +import csv +import re + +import datasets + + +logger = datasets.logging.get_logger(__name__) + + +_CITATION = """\ +@InProceedings{XLiu.etal:IWSDS2019, + author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser}, + title = {Benchmarking Natural Language Understanding Services for building Conversational Agents}, + booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)}, + month = {April}, + year = {2019}, + address = {Ortigia, Siracusa (SR), Italy}, + publisher = {Springer}, + pages = {xxx--xxx}, + url = {http://www.xx.xx/xx/} +} +""" + +# You can copy an official description +_DESCRIPTION = """\ +Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples (original dataset has 25716 examples) from 68 unique intents belonging to 18 scenarios. +""" + +_HOMEPAGE = "https://github.com/xliuhw/NLU-Evaluation-Data" + +_LICENSE = "Creative Commons Attribution 4.0 International License (CC BY 4.0)" + +_URL = "https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv" + +ANNOTATION_PATTERN = re.compile(r"\[(.+?)\s+\:+\s(.+?)\]") + + +def remove_annotations(text): + """Remove named entity annotations from text example. + + Examples are defined based on `answer_annotation` column since it has the least number + of Nans. However, this column contains patterns of annotation of the form: + + [named_entity : part_of_text] + + e.g. [time : five am], [date : this week] + + We identity them with regex rule and replace all occurrences with just part_of_text. + """ + return ANNOTATION_PATTERN.sub(r"\2", text) + + +def define_intent_name(scenario, intent): + """Intent name is defined as concatenation of `scenario` and `intent` + values. + + See Also: + https://github.com/xliuhw/NLU-Evaluation-Data/issues/5 + """ + return f"{scenario}_{intent}" + + +class NLUEvaluationData(datasets.GeneratorBasedBuilder): + """Raw part of NLU Evaluation Data.""" + + VERSION = datasets.Version("1.1.0") + + def _info(self): + features = datasets.Features( + { + "text": datasets.Value("string"), + "scenario": datasets.Value("string"), + "label": datasets.features.ClassLabel( + names=[ + "alarm_query", + "alarm_remove", + "alarm_set", + "audio_volume_down", + "audio_volume_mute", + "audio_volume_other", + "audio_volume_up", + "calendar_query", + "calendar_remove", + "calendar_set", + "cooking_query", + "cooking_recipe", + "datetime_convert", + "datetime_query", + "email_addcontact", + "email_query", + "email_querycontact", + "email_sendemail", + "general_affirm", + "general_commandstop", + "general_confirm", + "general_dontcare", + "general_explain", + "general_greet", + "general_joke", + "general_negate", + "general_praise", + "general_quirky", + "general_repeat", + "iot_cleaning", + "iot_coffee", + "iot_hue_lightchange", + "iot_hue_lightdim", + "iot_hue_lightoff", + "iot_hue_lighton", + "iot_hue_lightup", + "iot_wemo_off", + "iot_wemo_on", + "lists_createoradd", + "lists_query", + "lists_remove", + "music_dislikeness", + "music_likeness", + "music_query", + "music_settings", + "news_query", + "play_audiobook", + "play_game", + "play_music", + "play_podcasts", + "play_radio", + "qa_currency", + "qa_definition", + "qa_factoid", + "qa_maths", + "qa_stock", + "recommendation_events", + "recommendation_locations", + "recommendation_movies", + "social_post", + "social_query", + "takeaway_order", + "takeaway_query", + "transport_query", + "transport_taxi", + "transport_ticket", + "transport_traffic", + "weather_query", + ] + ), + } + ) + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + supervised_keys=None, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + """Returns SplitGenerators.""" + train_path = dl_manager.download_and_extract(_URL) + return [ + datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": train_path}), + ] + + def _generate_examples(self, filepath): + """Yields examples as (key, example) tuples.""" + with open(filepath, encoding="utf-8") as f: + csv_reader = csv.reader(f, quotechar='"', delimiter=";", quoting=csv.QUOTE_ALL, skipinitialspace=True) + # call next to skip header + next(csv_reader) + for id_, row in enumerate(csv_reader): + ( + userid, + answerid, + scenario, + intent, + status, + answer_annotation, + notes, + suggested_entities, + answer_normalised, + answer, + question, + ) = row + + # examples with empty answer are removed as part of the dataset + if answer_annotation == "null": + continue + + yield id_, { + "text": remove_annotations(answer_annotation), + "scenario": scenario, + "label": define_intent_name(scenario, intent), + }