From 5d486d6232ffe39895d87cabe85fe6fcbe48796d Mon Sep 17 00:00:00 2001 From: Dariusz Kajtoch Date: Mon, 12 Apr 2021 18:57:35 +0200 Subject: [PATCH 1/6] add nlu-evaluation-data with post processing logic --- .../nlu_evaluation_data.py | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 datasets/nlu_evaluation_data/nlu_evaluation_data.py diff --git a/datasets/nlu_evaluation_data/nlu_evaluation_data.py b/datasets/nlu_evaluation_data/nlu_evaluation_data.py new file mode 100644 index 00000000000..4c0cfd8b065 --- /dev/null +++ b/datasets/nlu_evaluation_data/nlu_evaluation_data.py @@ -0,0 +1,235 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TODO: Add a description here.""" + +from __future__ import absolute_import, division, print_function + +import csv +import re + +import datasets + + +logger = datasets.logging.get_logger(__name__) + + +_CITATION = """\ +@InProceedings{XLiu.etal:IWSDS2019, + author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser}, + title = {Benchmarking Natural Language Understanding Services for building Conversational Agents}, + booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)}, + month = {April}, + year = {2019}, + address = {Ortigia, Siracusa (SR), Italy}, + publisher = {Springer}, + pages = {xxx--xxx}, + url = {http://www.xx.xx/xx/} +} +""" + +# TODO: Add description of the dataset here +# You can copy an official description +_DESCRIPTION = """\ +Raw part of NLU Evaluation Data. +""" + +_HOMEPAGE = "https://github.com/xliuhw/NLU-Evaluation-Data" + +_LICENSE = "Creative Commons Attribution 4.0 International License (CC BY 4.0)" + +_URL = "https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv" + +ANNOTATION_PATTERN = re.compile(r"\[(.+?)\s+\:+\s(.+?)\]") + + +def remove_annotations(text): + """ + Remove named entity annotations from text example. + + Examples are defined based on `answer_annotation` column since it has the least number + of Nans. However, this column contains patterns of annotation of the form: + + [named_entity : part_of_text] + + e.g. [time : five am], [date : this week] + + We identity them with regex rule and replace all occurrences with just part_of_text. + """ + return ANNOTATION_PATTERN.sub(r"\2", text) + + +def define_intent_name(scenario, intent): + """ + Intent name is defined as concatenation of `scenario` and `intent` values. + + See Also: + https://github.com/xliuhw/NLU-Evaluation-Data/issues/5 + """ + return f"{scenario}_{intent}" + + +class NLUEvaluationDataset(datasets.GeneratorBasedBuilder): + """Raw part of NLU Evaluation Data.""" + + VERSION = datasets.Version("1.1.0") + + def _info(self): + features = datasets.Features( + { + "answer_annotation": datasets.Value("string"), + "scenario": datasets.Value("string"), + "intent": datasets.Value("string"), + } + ) + post_process_features = datasets.Features( + { + "text": datasets.Value("string"), + "scenario": datasets.Value("string"), + "label": datasets.features.ClassLabel( + names=[ + "alarm_query", + "alarm_remove", + "alarm_set", + "audio_volume_down", + "audio_volume_mute", + "audio_volume_other", + "audio_volume_up", + "calendar_query", + "calendar_remove", + "calendar_set", + "cooking_query", + "cooking_recipe", + "datetime_convert", + "datetime_query", + "email_addcontact", + "email_query", + "email_querycontact", + "email_sendemail", + "general_affirm", + "general_commandstop", + "general_confirm", + "general_dontcare", + "general_explain", + "general_greet", + "general_joke", + "general_negate", + "general_praise", + "general_quirky", + "general_repeat", + "iot_cleaning", + "iot_coffee", + "iot_hue_lightchange", + "iot_hue_lightdim", + "iot_hue_lightoff", + "iot_hue_lighton", + "iot_hue_lightup", + "iot_wemo_off", + "iot_wemo_on", + "lists_createoradd", + "lists_query", + "lists_remove", + "music_dislikeness", + "music_likeness", + "music_query", + "music_settings", + "news_query", + "play_audiobook", + "play_game", + "play_music", + "play_podcasts", + "play_radio", + "qa_currency", + "qa_definition", + "qa_factoid", + "qa_maths", + "qa_stock", + "recommendation_events", + "recommendation_locations", + "recommendation_movies", + "social_post", + "social_query", + "takeaway_order", + "takeaway_query", + "transport_query", + "transport_taxi", + "transport_ticket", + "transport_traffic", + "weather_query", + ] + ), + } + ) + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + supervised_keys=None, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + post_processed=datasets.info.PostProcessedInfo( + features=post_process_features, + ), + ) + + def _split_generators(self, dl_manager): + """Returns SplitGenerators.""" + train_path = dl_manager.download_and_extract(_URL) + return [ + datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": train_path}), + ] + + def _post_process(self, dataset, resources_paths): + # remove empty examples + dataset = dataset.filter(lambda ex: True if ex["answer_annotation"] != "null" else False, with_indices=False) + # remove annotations from examples and define label + dataset = dataset.map( + lambda ex: dict( + ex, + text=remove_annotations(ex["answer_annotation"]), + label=self.info.post_processed.features["label"].encode_example( + define_intent_name(ex["scenario"], ex["intent"]) + ), + ), + batched=False, + with_indices=False, + features=datasets.Features(self.info.features, **self.info.post_processed.features), + ) + # drop unused columns + dataset = dataset.remove_columns(["answer_annotation", "intent"]) + return dataset + + def _generate_examples(self, filepath): + """ Yields examples as (key, example) tuples. """ + with open(filepath, encoding="utf-8") as f: + csv_reader = csv.reader(f, quotechar='"', delimiter=";", quoting=csv.QUOTE_ALL, skipinitialspace=True) + # call next to skip header + next(csv_reader) + for id_, row in enumerate(csv_reader): + ( + userid, + answerid, + scenario, + intent, + status, + answer_annotation, + notes, + suggested_entities, + answer_normalised, + answer, + question, + ) = row + + yield id_, {"answer_annotation": answer_annotation, "scenario": scenario, "intent": intent} From 601e4702dcfe19d55f79776375919257f0f0e03a Mon Sep 17 00:00:00 2001 From: Dariusz Kajtoch Date: Mon, 19 Apr 2021 18:43:43 +0200 Subject: [PATCH 2/6] add short dataset description --- datasets/nlu_evaluation_data/nlu_evaluation_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/datasets/nlu_evaluation_data/nlu_evaluation_data.py b/datasets/nlu_evaluation_data/nlu_evaluation_data.py index 4c0cfd8b065..a527117bebe 100644 --- a/datasets/nlu_evaluation_data/nlu_evaluation_data.py +++ b/datasets/nlu_evaluation_data/nlu_evaluation_data.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""TODO: Add a description here.""" +"""NLU Evaluation Data.""" from __future__ import absolute_import, division, print_function @@ -39,10 +39,9 @@ } """ -# TODO: Add description of the dataset here # You can copy an official description _DESCRIPTION = """\ -Raw part of NLU Evaluation Data. +Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios. """ _HOMEPAGE = "https://github.com/xliuhw/NLU-Evaluation-Data" From d9b813ddbfb32ee8381348520c9322d90f062f8d Mon Sep 17 00:00:00 2001 From: Dariusz Kajtoch Date: Mon, 19 Apr 2021 18:44:03 +0200 Subject: [PATCH 3/6] include dataset infos and dummy data --- datasets/nlu_evaluation_data/dataset_infos.json | 1 + .../dummy/1.1.0/dummy_data.zip | Bin 0 -> 618 bytes 2 files changed, 1 insertion(+) create mode 100644 datasets/nlu_evaluation_data/dataset_infos.json create mode 100644 datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip diff --git a/datasets/nlu_evaluation_data/dataset_infos.json b/datasets/nlu_evaluation_data/dataset_infos.json new file mode 100644 index 00000000000..e580ad7554e --- /dev/null +++ b/datasets/nlu_evaluation_data/dataset_infos.json @@ -0,0 +1 @@ +{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"answer_annotation": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "intent": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_dataset", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1800744, "num_examples": 25716, "dataset_name": "nlu_evaluation_dataset"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1800744, "size_in_bytes": 7668183}} \ No newline at end of file diff --git a/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip b/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..141fcec4bcdac4a4cf0fc913b23e8a2619fc82c3 GIT binary patch literal 618 zcmWIWW@Zs#00FV7lY_tvD8UV+Q%ZAlE8|lVOA_@1pb9y_3J<(iH*{fSVA#*hz+i}~ z(9b7S*9E9U*CRhSRo5jyH!(9$*D)_Iza+6FHAUAkCr2;2xNPlYLw{yNp4RfZ)|-J# zn-T=Sb@b=EV>R}?w^ZZ=m$F2>R`Yz$BWE!s zc+2DunJ;wCKQ;V1_nBLLTW!J3$A{7n*8PfI#{BEr&9M8s(_E$bPOUE1EqVIY{`jvI zf~SL?ul`+rY>VsOcQe%D1H2iTM3`~M9wZ_K7~VR9SSWD`(}+DbAsQJNmNaUjY9uah V1H4(;Kt?bD;S3 Date: Mon, 19 Apr 2021 18:44:14 +0200 Subject: [PATCH 4/6] add dataset readme --- datasets/nlu_evaluation_data/README.md | 271 +++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 datasets/nlu_evaluation_data/README.md diff --git a/datasets/nlu_evaluation_data/README.md b/datasets/nlu_evaluation_data/README.md new file mode 100644 index 00000000000..ce33de865a7 --- /dev/null +++ b/datasets/nlu_evaluation_data/README.md @@ -0,0 +1,271 @@ +--- +annotations_creators: +- expert-generated +extended: +- original +language_creators: +- expert-generated +languages: +- en +licenses: +- cc-by-4.0 +multilinguality: +- monolingual +size_categories: +- 10K To build the NLU component we collected real user data via Amazon Mechanical +Turk (AMT). We designed tasks where the Turker’s goal was to answer questions +about how people would interact with the home robot, in a wide range of scenarios +designed in advance, namely: alarm, audio, audiobook, calendar, cooking, datetime, +email, game, general, IoT, lists, music, news, podcasts, general Q&A, radio, recommendations, social, food takeaway, transport, and weather. +The questions put to Turkers were designed to capture the different requests +within each given scenario. +In the ‘calendar’ scenario, for example, these pre-designed intents were included: ‘set event’, ‘delete event’ and ‘query event’. +An example question for intent ‘set event’ is: “How would you ask your PDA to schedule a meeting with someone?” for which a user’s answer example was “Schedule +a chat with Adam on Thursday afternoon”. +The Turkers would then type in their answers to these questions and select possible entities from the pre-designed suggested entities list for each of their answers.The Turkers didn’t always follow the +instructions fully, e.g. for the specified ‘delete event’ Intent, an answer was: “PDA +what is my next event?”; which clearly belongs to ‘query event’ Intent. +We have manually corrected all such errors either during post-processing or the subsequent annotations. + +#### Who are the annotators? + +[More Information Needed] + +### Personal and Sensitive Information + +[More Information Needed] + +## Considerations for Using the Data + +### Social Impact of Dataset + +The purpose of this dataset it to help develop better intent detection systems. + +### Discussion of Biases + +[More Information Needed] + +### Other Known Limitations + +[More Information Needed] + +## Additional Information + +### Dataset Curators + +[More Information Needed] + +### Licensing Information + +Creative Commons Attribution 4.0 International License (CC BY 4.0) + +### Citation Information +``` +@InProceedings{XLiu.etal:IWSDS2019, + author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser}, + title = {Benchmarking Natural Language Understanding Services for building Conversational Agents}, + booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)}, + month = {April}, + year = {2019}, + address = {Ortigia, Siracusa (SR), Italy}, + publisher = {Springer}, + pages = {xxx--xxx}, + url = {http://www.xx.xx/xx/} +} +``` +### Contributions + +Thanks to [@dkajtoch](https://github.com/dkajtoch) for adding this dataset. \ No newline at end of file From e8ae7d332df4ae5e3a0ccdcdfd1f7dbf66336de6 Mon Sep 17 00:00:00 2001 From: Dariusz Kajtoch <32985207+dkajtoch@users.noreply.github.com> Date: Wed, 21 Apr 2021 17:26:55 +0200 Subject: [PATCH 5/6] move processing from _post_process to _generate_examples except dataset filter --- datasets/nlu_evaluation_data/README.md | 18 +++------- .../nlu_evaluation_data/dataset_infos.json | 2 +- .../nlu_evaluation_data.py | 35 ++++--------------- 3 files changed, 13 insertions(+), 42 deletions(-) diff --git a/datasets/nlu_evaluation_data/README.md b/datasets/nlu_evaluation_data/README.md index ce33de865a7..5cd2e850c03 100644 --- a/datasets/nlu_evaluation_data/README.md +++ b/datasets/nlu_evaluation_data/README.md @@ -205,19 +205,11 @@ is much larger and contains 68 intents from 18 scenarios, which is much larger t #### Annotation process -> To build the NLU component we collected real user data via Amazon Mechanical -Turk (AMT). We designed tasks where the Turker’s goal was to answer questions -about how people would interact with the home robot, in a wide range of scenarios -designed in advance, namely: alarm, audio, audiobook, calendar, cooking, datetime, -email, game, general, IoT, lists, music, news, podcasts, general Q&A, radio, recommendations, social, food takeaway, transport, and weather. -The questions put to Turkers were designed to capture the different requests -within each given scenario. +> To build the NLU component we collected real user data via Amazon Mechanical Turk (AMT). We designed tasks where the Turker’s goal was to answer questions about how people would interact with the home robot, in a wide range of scenarios designed in advance, namely: alarm, audio, audiobook, calendar, cooking, datetime, email, game, general, IoT, lists, music, news, podcasts, general Q&A, radio, recommendations, social, food takeaway, transport, and weather. +The questions put to Turkers were designed to capture the different requests within each given scenario. In the ‘calendar’ scenario, for example, these pre-designed intents were included: ‘set event’, ‘delete event’ and ‘query event’. -An example question for intent ‘set event’ is: “How would you ask your PDA to schedule a meeting with someone?” for which a user’s answer example was “Schedule -a chat with Adam on Thursday afternoon”. -The Turkers would then type in their answers to these questions and select possible entities from the pre-designed suggested entities list for each of their answers.The Turkers didn’t always follow the -instructions fully, e.g. for the specified ‘delete event’ Intent, an answer was: “PDA -what is my next event?”; which clearly belongs to ‘query event’ Intent. +An example question for intent ‘set event’ is: “How would you ask your PDA to schedule a meeting with someone?” for which a user’s answer example was “Schedule a chat with Adam on Thursday afternoon”. +The Turkers would then type in their answers to these questions and select possible entities from the pre-designed suggested entities list for each of their answers.The Turkers didn’t always follow the instructions fully, e.g. for the specified ‘delete event’ Intent, an answer was: “PDA what is my next event?”; which clearly belongs to ‘query event’ Intent. We have manually corrected all such errors either during post-processing or the subsequent annotations. #### Who are the annotators? @@ -268,4 +260,4 @@ Creative Commons Attribution 4.0 International License (CC BY 4.0) ``` ### Contributions -Thanks to [@dkajtoch](https://github.com/dkajtoch) for adding this dataset. \ No newline at end of file +Thanks to [@dkajtoch](https://github.com/dkajtoch) for adding this dataset. diff --git a/datasets/nlu_evaluation_data/dataset_infos.json b/datasets/nlu_evaluation_data/dataset_infos.json index e580ad7554e..b830ae7966c 100644 --- a/datasets/nlu_evaluation_data/dataset_infos.json +++ b/datasets/nlu_evaluation_data/dataset_infos.json @@ -1 +1 @@ -{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"answer_annotation": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "intent": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_dataset", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1800744, "num_examples": 25716, "dataset_name": "nlu_evaluation_dataset"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1800744, "size_in_bytes": 7668183}} \ No newline at end of file +{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_data", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1447966, "num_examples": 25716, "dataset_name": "nlu_evaluation_data"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1447966, "size_in_bytes": 7315405}} \ No newline at end of file diff --git a/datasets/nlu_evaluation_data/nlu_evaluation_data.py b/datasets/nlu_evaluation_data/nlu_evaluation_data.py index a527117bebe..b42c7b731b6 100644 --- a/datasets/nlu_evaluation_data/nlu_evaluation_data.py +++ b/datasets/nlu_evaluation_data/nlu_evaluation_data.py @@ -79,20 +79,13 @@ def define_intent_name(scenario, intent): return f"{scenario}_{intent}" -class NLUEvaluationDataset(datasets.GeneratorBasedBuilder): +class NLUEvaluationData(datasets.GeneratorBasedBuilder): """Raw part of NLU Evaluation Data.""" VERSION = datasets.Version("1.1.0") def _info(self): features = datasets.Features( - { - "answer_annotation": datasets.Value("string"), - "scenario": datasets.Value("string"), - "intent": datasets.Value("string"), - } - ) - post_process_features = datasets.Features( { "text": datasets.Value("string"), "scenario": datasets.Value("string"), @@ -178,9 +171,6 @@ def _info(self): homepage=_HOMEPAGE, license=_LICENSE, citation=_CITATION, - post_processed=datasets.info.PostProcessedInfo( - features=post_process_features, - ), ) def _split_generators(self, dl_manager): @@ -192,22 +182,7 @@ def _split_generators(self, dl_manager): def _post_process(self, dataset, resources_paths): # remove empty examples - dataset = dataset.filter(lambda ex: True if ex["answer_annotation"] != "null" else False, with_indices=False) - # remove annotations from examples and define label - dataset = dataset.map( - lambda ex: dict( - ex, - text=remove_annotations(ex["answer_annotation"]), - label=self.info.post_processed.features["label"].encode_example( - define_intent_name(ex["scenario"], ex["intent"]) - ), - ), - batched=False, - with_indices=False, - features=datasets.Features(self.info.features, **self.info.post_processed.features), - ) - # drop unused columns - dataset = dataset.remove_columns(["answer_annotation", "intent"]) + dataset = dataset.filter(lambda ex: True if ex["text"] != "null" else False, with_indices=False) return dataset def _generate_examples(self, filepath): @@ -231,4 +206,8 @@ def _generate_examples(self, filepath): question, ) = row - yield id_, {"answer_annotation": answer_annotation, "scenario": scenario, "intent": intent} + yield id_, { + "text": remove_annotations(answer_annotation), + "scenario": scenario, + "label": define_intent_name(scenario, intent), + } From 3b175e9c9ea533fbbf152f39f680b377ef7d5dce Mon Sep 17 00:00:00 2001 From: Dariusz Kajtoch Date: Fri, 23 Apr 2021 16:56:34 +0200 Subject: [PATCH 6/6] move filtering of empty rows to _generate_examples --- datasets/nlu_evaluation_data/README.md | 2 +- .../nlu_evaluation_data/dataset_infos.json | 2 +- .../dummy/1.1.0/dummy_data.zip | Bin 618 -> 618 bytes .../nlu_evaluation_data.py | 20 ++++++++---------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/datasets/nlu_evaluation_data/README.md b/datasets/nlu_evaluation_data/README.md index 5cd2e850c03..9d94e067f77 100644 --- a/datasets/nlu_evaluation_data/README.md +++ b/datasets/nlu_evaluation_data/README.md @@ -66,7 +66,7 @@ task_ids: Dataset with short utterances from conversational domain annotated with their corresponding intents and scenarios. -It has 25 715 non-zero examples belonging to 18 scenarios and 68 intents. +It has 25 715 non-zero examples (original dataset has 25716 examples) belonging to 18 scenarios and 68 intents. Originally, the dataset was crowd-sourced and annotated with both intents and named entities in order to evaluate commercial NLU systems such as RASA, IBM's Watson, Microsoft's LUIS and Google's Dialogflow. **This version of the dataset only includes intent annotations!** diff --git a/datasets/nlu_evaluation_data/dataset_infos.json b/datasets/nlu_evaluation_data/dataset_infos.json index b830ae7966c..8d3364b59a5 100644 --- a/datasets/nlu_evaluation_data/dataset_infos.json +++ b/datasets/nlu_evaluation_data/dataset_infos.json @@ -1 +1 @@ -{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_data", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1447966, "num_examples": 25716, "dataset_name": "nlu_evaluation_data"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1447966, "size_in_bytes": 7315405}} \ No newline at end of file +{"default": {"description": "Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples (original dataset has 25716 examples) from 68 unique intents belonging to 18 scenarios.\n", "citation": "@InProceedings{XLiu.etal:IWSDS2019,\n author = {Xingkun Liu, Arash Eshghi, Pawel Swietojanski and Verena Rieser},\n title = {Benchmarking Natural Language Understanding Services for building Conversational Agents},\n booktitle = {Proceedings of the Tenth International Workshop on Spoken Dialogue Systems Technology (IWSDS)},\n month = {April},\n year = {2019},\n address = {Ortigia, Siracusa (SR), Italy},\n publisher = {Springer},\n pages = {xxx--xxx},\n url = {http://www.xx.xx/xx/}\n}\n", "homepage": "https://github.com/xliuhw/NLU-Evaluation-Data", "license": "Creative Commons Attribution 4.0 International License (CC BY 4.0)", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "post_processed": {"features": {"text": {"dtype": "string", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "label": {"num_classes": 68, "names": ["alarm_query", "alarm_remove", "alarm_set", "audio_volume_down", "audio_volume_mute", "audio_volume_other", "audio_volume_up", "calendar_query", "calendar_remove", "calendar_set", "cooking_query", "cooking_recipe", "datetime_convert", "datetime_query", "email_addcontact", "email_query", "email_querycontact", "email_sendemail", "general_affirm", "general_commandstop", "general_confirm", "general_dontcare", "general_explain", "general_greet", "general_joke", "general_negate", "general_praise", "general_quirky", "general_repeat", "iot_cleaning", "iot_coffee", "iot_hue_lightchange", "iot_hue_lightdim", "iot_hue_lightoff", "iot_hue_lighton", "iot_hue_lightup", "iot_wemo_off", "iot_wemo_on", "lists_createoradd", "lists_query", "lists_remove", "music_dislikeness", "music_likeness", "music_query", "music_settings", "news_query", "play_audiobook", "play_game", "play_music", "play_podcasts", "play_radio", "qa_currency", "qa_definition", "qa_factoid", "qa_maths", "qa_stock", "recommendation_events", "recommendation_locations", "recommendation_movies", "social_post", "social_query", "takeaway_order", "takeaway_query", "transport_query", "transport_taxi", "transport_ticket", "transport_traffic", "weather_query"], "names_file": null, "id": null, "_type": "ClassLabel"}}, "resources_checksums": {"train": {}}}, "supervised_keys": null, "builder_name": "nlu_evaluation_data", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1447941, "num_examples": 25715, "dataset_name": "nlu_evaluation_data"}}, "download_checksums": {"https://raw.githubusercontent.com/xliuhw/NLU-Evaluation-Data/master/AnnotatedData/NLU-Data-Home-Domain-Annotated-All.csv": {"num_bytes": 5867439, "checksum": "5f6dbf6d38fc111217924945ac59c554e0b926d5aa836ecdd0d089d2ca48e1d9"}}, "download_size": 5867439, "post_processing_size": 0, "dataset_size": 1447941, "size_in_bytes": 7315380}} \ No newline at end of file diff --git a/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip b/datasets/nlu_evaluation_data/dummy/1.1.0/dummy_data.zip index 141fcec4bcdac4a4cf0fc913b23e8a2619fc82c3..2e72bebc733c7e1c42bfb34620df983d17c3f31c 100644 GIT binary patch delta 45 rcmaFG@`{Bwz?+#xgaHKfTBc9rRbvKH8_gy$g6PSo8O^{n3sV{Z4%7^v delta 45 rcmaFG@`{Bwz?+#xgaHJ^rcR#7tHunZHkwUh1ksaEGn#>E7N#@+3wI22 diff --git a/datasets/nlu_evaluation_data/nlu_evaluation_data.py b/datasets/nlu_evaluation_data/nlu_evaluation_data.py index b42c7b731b6..07b1c1c0767 100644 --- a/datasets/nlu_evaluation_data/nlu_evaluation_data.py +++ b/datasets/nlu_evaluation_data/nlu_evaluation_data.py @@ -41,7 +41,7 @@ # You can copy an official description _DESCRIPTION = """\ -Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples from 68 unique intents belonging to 18 scenarios. +Raw part of NLU Evaluation Data. It contains 25 715 non-empty examples (original dataset has 25716 examples) from 68 unique intents belonging to 18 scenarios. """ _HOMEPAGE = "https://github.com/xliuhw/NLU-Evaluation-Data" @@ -54,8 +54,7 @@ def remove_annotations(text): - """ - Remove named entity annotations from text example. + """Remove named entity annotations from text example. Examples are defined based on `answer_annotation` column since it has the least number of Nans. However, this column contains patterns of annotation of the form: @@ -70,8 +69,8 @@ def remove_annotations(text): def define_intent_name(scenario, intent): - """ - Intent name is defined as concatenation of `scenario` and `intent` values. + """Intent name is defined as concatenation of `scenario` and `intent` + values. See Also: https://github.com/xliuhw/NLU-Evaluation-Data/issues/5 @@ -180,13 +179,8 @@ def _split_generators(self, dl_manager): datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": train_path}), ] - def _post_process(self, dataset, resources_paths): - # remove empty examples - dataset = dataset.filter(lambda ex: True if ex["text"] != "null" else False, with_indices=False) - return dataset - def _generate_examples(self, filepath): - """ Yields examples as (key, example) tuples. """ + """Yields examples as (key, example) tuples.""" with open(filepath, encoding="utf-8") as f: csv_reader = csv.reader(f, quotechar='"', delimiter=";", quoting=csv.QUOTE_ALL, skipinitialspace=True) # call next to skip header @@ -206,6 +200,10 @@ def _generate_examples(self, filepath): question, ) = row + # examples with empty answer are removed as part of the dataset + if answer_annotation == "null": + continue + yield id_, { "text": remove_annotations(answer_annotation), "scenario": scenario,