From 3fc2ef2f81895eeb3fe815c283ed08a4b815de93 Mon Sep 17 00:00:00 2001 From: Consensus-NLP Date: Thu, 18 May 2023 16:04:18 +0330 Subject: [PATCH 1/2] added consensus_summary registery and data --- evals/registry/data/consensus_summary/samples.jsonl | 3 +++ evals/registry/evals/consensus_summary.yaml | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 evals/registry/data/consensus_summary/samples.jsonl create mode 100644 evals/registry/evals/consensus_summary.yaml diff --git a/evals/registry/data/consensus_summary/samples.jsonl b/evals/registry/data/consensus_summary/samples.jsonl new file mode 100644 index 0000000000..7605841b3e --- /dev/null +++ b/evals/registry/data/consensus_summary/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3573fca21526acb8e65cba1e6e88ae6877856454ab224dda4bd04828527c1299 +size 495294 diff --git a/evals/registry/evals/consensus_summary.yaml b/evals/registry/evals/consensus_summary.yaml new file mode 100644 index 0000000000..886f6c7c10 --- /dev/null +++ b/evals/registry/evals/consensus_summary.yaml @@ -0,0 +1,9 @@ +consensus_summary: + id: consensus_summary.dev.v0 + description: Utilize the model's ability to produce a Consensus in response to a scientific inquiry. + metrics: [accuracy] + +consensus_summary.dev.v0: + class: evals.elsuite.basic.includes:Includes + args: + samples_jsonl: consensus_summary/samples.jsonl From feeb677a88dc4ea1fd1b68bb67dc447a59eaef55 Mon Sep 17 00:00:00 2001 From: Consensus-NLP Date: Sun, 11 Jun 2023 13:01:44 +0330 Subject: [PATCH 2/2] Update consensus_summary.yaml added modelgraded fact --- evals/registry/evals/consensus_summary.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/registry/evals/consensus_summary.yaml b/evals/registry/evals/consensus_summary.yaml index 886f6c7c10..26745469a0 100644 --- a/evals/registry/evals/consensus_summary.yaml +++ b/evals/registry/evals/consensus_summary.yaml @@ -4,6 +4,8 @@ consensus_summary: metrics: [accuracy] consensus_summary.dev.v0: - class: evals.elsuite.basic.includes:Includes + class: evals.elsuite.modelgraded.classify:ModelBasedClassify args: samples_jsonl: consensus_summary/samples.jsonl + eval_type: cot_classify + modelgraded_spec: fact