From c921635b77c09d984246e7f929767b69e35b8395 Mon Sep 17 00:00:00 2001 From: Iges-Institut Date: Fri, 2 Jun 2023 12:53:32 +0200 Subject: [PATCH 1/4] contributing a custom eval to the repository. The model is tasked to extract text spans specifying the population demographic from abstract of clinical drug trials. --- .../data/population_span_extraction/samples.jsonl | 3 +++ evals/registry/evals/population_span_extraction.yaml | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 evals/registry/data/population_span_extraction/samples.jsonl create mode 100644 evals/registry/evals/population_span_extraction.yaml diff --git a/evals/registry/data/population_span_extraction/samples.jsonl b/evals/registry/data/population_span_extraction/samples.jsonl new file mode 100644 index 0000000000..7c58463994 --- /dev/null +++ b/evals/registry/data/population_span_extraction/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75d73da796eacae15d03b59da8578f496ca3b6450d9a77cb1d28670e558e628 +size 2323327 diff --git a/evals/registry/evals/population_span_extraction.yaml b/evals/registry/evals/population_span_extraction.yaml new file mode 100644 index 0000000000..324e382bfe --- /dev/null +++ b/evals/registry/evals/population_span_extraction.yaml @@ -0,0 +1,12 @@ +# Register the eval by adding a file to evals/registry/evals/.yaml +population_span_extraction: + id: population_span_extraction.dev.v0 + metrics: [accuracy] + +population_span_extraction.dev.v0: + # Assuming that the response can contain "significant variation" --> modelbased classify + class: evals.elsuite.modelgraded.classify:ModelBasedClassify + args: + samples_jsonl: population_span_extraction/samples.jsonl + eval_type: cot_classify + modelgraded_spec: fact \ No newline at end of file From b0d6c2e0eabfd3d467c950bc2cf5d9a5a6e7362b Mon Sep 17 00:00:00 2001 From: Iges-Institut Date: Fri, 2 Jun 2023 13:13:24 +0200 Subject: [PATCH 2/4] Made sure to move the JSON file to Git LFS before submitting a PR --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 9f45a92b72..1bed20193a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ evals/registry/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text +evals/registry/data/population_span_extraction/samples.jsonl filter=lfs diff=lfs merge=lfs -text From f895999072546f7cf3148d8bd0638f7c04ea1ce3 Mon Sep 17 00:00:00 2001 From: Iges-Institut Date: Wed, 7 Jun 2023 14:16:13 +0200 Subject: [PATCH 3/4] reverted changes to .gitattribute --- .gitattributes | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 1bed20193a..48163b46e0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ -evals/registry/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text -evals/registry/data/population_span_extraction/samples.jsonl filter=lfs diff=lfs merge=lfs -text +evals/registry/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text \ No newline at end of file From a75f6b68c5c9528a446c4c115e97c705e37cbe3c Mon Sep 17 00:00:00 2001 From: Iges-Institut Date: Wed, 7 Jun 2023 14:20:56 +0200 Subject: [PATCH 4/4] added a proper description for the eval in the .yaml file --- evals/registry/evals/population_span_extraction.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/registry/evals/population_span_extraction.yaml b/evals/registry/evals/population_span_extraction.yaml index 324e382bfe..a6a3b62e14 100644 --- a/evals/registry/evals/population_span_extraction.yaml +++ b/evals/registry/evals/population_span_extraction.yaml @@ -2,6 +2,7 @@ population_span_extraction: id: population_span_extraction.dev.v0 metrics: [accuracy] + description: The model is shown abstracts of clinical drug trials and tasked with extracting the text spans that specify the population demographic of the shown abstract. The population demographic can be but is not necessarily specified in multiple seperate spans. population_span_extraction.dev.v0: # Assuming that the response can contain "significant variation" --> modelbased classify