From bf6ba21299e8b9ac1f39bcc2e7a201cd5cbc4473 Mon Sep 17 00:00:00 2001 From: Iges-Institut Date: Thu, 8 Jun 2023 14:45:53 +0200 Subject: [PATCH] contributing a custum eval to the repository. description of the eval is in the registry file. --- .../data/population_span_extraction/samples.jsonl | 3 +++ evals/registry/evals/population_span_extraction.yaml | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 evals/registry/data/population_span_extraction/samples.jsonl create mode 100644 evals/registry/evals/population_span_extraction.yaml diff --git a/evals/registry/data/population_span_extraction/samples.jsonl b/evals/registry/data/population_span_extraction/samples.jsonl new file mode 100644 index 0000000000..5d63a3d894 --- /dev/null +++ b/evals/registry/data/population_span_extraction/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dce5e57cb3fe18df839314fcc0047ae4db9cc97af6f7cdcfd81c23ec788c587 +size 1076022 diff --git a/evals/registry/evals/population_span_extraction.yaml b/evals/registry/evals/population_span_extraction.yaml new file mode 100644 index 0000000000..0e6cebf2eb --- /dev/null +++ b/evals/registry/evals/population_span_extraction.yaml @@ -0,0 +1,11 @@ +population_span_extraction: + id: population_span_extraction.dev.v0 + description: The model is shown abstracts of clinical drug trials and tasked with extracting the text spans that specify the population demographic of the shown abstract. The population demographic can be but is not necessarily specified in multiple seperate spans. + metrics: [accuracy] + +population_span_extraction.dev.v0: + class: evals.elsuite.modelgraded.classify:ModelBasedClassify + args: + samples_jsonl: population_span_extraction/samples.jsonl + eval_type: cot_classify + modelgraded_spec: fact \ No newline at end of file