Adding Basic Skills to in-loop eval (#7)

* basic skills task

* basic skills task

* basic skills coding

* basic_skills_arithmetic

* basic_skills_string_operations

* basic_skills_coding

* basic_skills_pattern

* basic_skills_common_knowledge

* basic_skills_logical_reasoning

* adding tasks to tasks.py

* requests.jsonl.gz

* requests.jsonl.gz

* lint

* prep for release v0.7.2

---------

Co-authored-by: David Heineman <[email protected]>

Author: rlebras

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [v0.7.2](https://github.com/allenai/OLMo-in-loop-evals/releases/tag/v0.7.2) - 2025-05-16
+
+- Add basic skills evals
+
 ## [v0.7.1](https://github.com/allenai/OLMo-in-loop-evals/releases/tag/v0.7.1) - 2025-04-02
 
 - Fix normalization to match the OLMES standard

src/olmo_eval/oe_eval_tasks/basic_skills_arithmetic/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_arithmetic",
+  "task_hash": "56711b967c78d896ef51ba00aef5cfb0",
+  "task_config": {
+    "dataset_path": "basic_skills_arithmetic",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_arithmetic:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "arithmetic",
+    "task_name": "basic_skills_arithmetic",
+    "version": 0,
+    "task_core": "basic_skills_arithmetic"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

src/olmo_eval/oe_eval_tasks/basic_skills_arithmetic/rc_5shot/requests.jsonl.gz

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_coding",
+  "task_hash": "d748d1d8ba506d3d234eed529ef62c3e",
+  "task_config": {
+    "dataset_path": "basic_skills_coding",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_coding:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "coding",
+    "task_name": "basic_skills_coding",
+    "version": 0,
+    "task_core": "basic_skills_coding"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

src/olmo_eval/oe_eval_tasks/basic_skills_coding/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_common_knowledge",
+  "task_hash": "51e88e759602f9085a8c779da375d833",
+  "task_config": {
+    "dataset_path": "basic_skills_common_knowledge",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_common_knowledge:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "common_knowledge",
+    "task_name": "basic_skills_common_knowledge",
+    "version": 0,
+    "task_core": "basic_skills_common_knowledge"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

src/olmo_eval/oe_eval_tasks/basic_skills_coding/rc_5shot/requests.jsonl.gz

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_logical_reasoning",
+  "task_hash": "a3d406a2f4224604b7e6bbf68050691d",
+  "task_config": {
+    "dataset_path": "basic_skills_logical_reasoning",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_logical_reasoning:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "logical_reasoning",
+    "task_name": "basic_skills_logical_reasoning",
+    "version": 0,
+    "task_core": "basic_skills_logical_reasoning"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

src/olmo_eval/oe_eval_tasks/basic_skills_common_knowledge/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_pattern",
+  "task_hash": "67983750bfb70a3b5cc34dcd67ee3c6a",
+  "task_config": {
+    "dataset_path": "basic_skills_pattern",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_pattern:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "pattern",
+    "task_name": "basic_skills_pattern",
+    "version": 0,
+    "task_core": "basic_skills_pattern"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}