diff --git a/evals/registry/data/sindarin_fluency/sindarin_nouns.jsonl b/evals/registry/data/sindarin_fluency/sindarin_nouns.jsonl new file mode 100644 index 0000000000..a58c606737 --- /dev/null +++ b/evals/registry/data/sindarin_fluency/sindarin_nouns.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1511df3846e871f222e7d7f00f23f2c454c1a055029203d8feaa05998486ee23 +size 41437 diff --git a/evals/registry/evals/sindarin-fluency.yaml b/evals/registry/evals/sindarin-fluency.yaml new file mode 100644 index 0000000000..280455c2b7 --- /dev/null +++ b/evals/registry/evals/sindarin-fluency.yaml @@ -0,0 +1,8 @@ +sindarin-fluency: + id: sindarin-fluency.dev.v0 + description: Tests the model's ability to accurately translate Sindarin to English. + metrics: [accuracy] +sindarin-fluency.dev.v0: + class: evals.elsuite.basic.includes:Includes + args: + samples_jsonl: sindarin_fluency/sindarin_nouns.jsonl \ No newline at end of file