diff --git a/evals/registry/data/soc_codes/samples.jsonl b/evals/registry/data/soc_codes/samples.jsonl new file mode 100644 index 0000000000..f46dd2a212 --- /dev/null +++ b/evals/registry/data/soc_codes/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:420b2cb1fe3a5536b41889f3506dc48d0cbab25302eea4b39890b6590d170619 +size 795740 diff --git a/evals/registry/evals/soc_codes.yaml b/evals/registry/evals/soc_codes.yaml new file mode 100644 index 0000000000..695729bbb9 --- /dev/null +++ b/evals/registry/evals/soc_codes.yaml @@ -0,0 +1,11 @@ +soc_codes: + id: soc_codes.dev.v0 + description: Eval that checks the models ability to classify a job title into a SOC codes issued by the American Bureau of Labor Statistics + metrics: [accuracy] + +soc_codes.dev.v0: + class: evals.elsuite.modelgraded.classify:ModelBasedClassify + args: + samples_jsonl: soc_codes/samples.jsonl + eval_type: classify + modelgraded_spec: fact