Expand ${var} in benchcomp variant env (#3090)

The values of environment variables in the benchcomp configuration file
can now contain strings of the form '${var}'. Benchcomp will replace
these strings with the value of the environment variable 'var'. This is
intended to allow users to have several benchcomp variants, each of
which differs only in the environment.

This fixes #2981.

Author: karkhaz

docs/src/benchcomp-conf.md

@@ -4,6 +4,26 @@
 This page lists the different visualizations that are available.
 
 
+## Variants
+
+A *variant* is a single invocation of a benchmark suite. Benchcomp runs several
+variants, so that their performance can be compared later. A variant consists of
+a command-line argument, working directory, and environment. Benchcomp invokes
+the command using the operating system environment, updated with the keys and
+values in `env`. If any values in `env` contain strings of the form `${var}`,
+Benchcomp expands them to the value of the environment variable `$var`.
+
+```yaml
+variants:
+    variant_1:
+        config:
+            command_line: echo "Hello, world"
+            directory: /tmp
+            env:
+              PATH: /my/local/directory:${PATH}
+```
+
+
 ## Built-in visualizations
 
 The following visualizations are available; these can be added to the `visualize` list of `benchcomp.yaml`.

tools/benchcomp/benchcomp/entry/run.py

@@ -13,6 +13,7 @@
 import logging
 import os
 import pathlib
+import re
 import shutil
 import subprocess
 import typing
@@ -53,9 +54,10 @@ def __post_init__(self):
         else:
             self.working_copy = pathlib.Path(self.directory)
 
+
     def __call__(self):
-        env = dict(os.environ)
-        env.update(self.env)
+        update_environment_with = _EnvironmentUpdater()
+        env = update_environment_with(self.env)
 
         if self.copy_benchmarks_dir:
             shutil.copytree(
@@ -128,6 +130,44 @@ def __call__(self):
         tmp_symlink.rename(self.out_symlink)
 
 
+
+@dataclasses.dataclass
+class _EnvironmentUpdater:
+    """Update the OS environment with keys and values containing variables
+
+    When called, this class returns the operating environment updated with new
+    keys and values. The values can contain variables of the form '${var_name}'.
+    The class evaluates those variables using values already in the environment.
+    """
+
+    os_environment: dict = dataclasses.field(
+        default_factory=lambda : dict(os.environ))
+    pattern: re.Pattern = re.compile(r"\$\{(\w+?)\}")
+
+
+    def _evaluate(self, key, value):
+        """Evaluate all ${var} in value using self.os_environment"""
+        old_value = value
+
+        for variable in re.findall(self.pattern, value):
+            if variable not in self.os_environment:
+                logging.error(
+                    "Couldn't evaluate ${%s} in the value '%s' for environment "
+                    "variable '%s'. Ensure the environment variable $%s is set",
+                    variable, old_value, key, variable)
+                sys.exit(1)
+            value = re.sub(
+                r"\$\{" + variable + "\}", self.os_environment[variable], value)
+        return value
+
+
+    def __call__(self, new_environment):
+        ret = dict(self.os_environment)
+        for key, value in new_environment.items():
+            ret[key] = self._evaluate(key, value)
+        return ret
+
+
 def get_default_out_symlink():
     return "latest"
 

tools/benchcomp/test/test_regression.py

@@ -662,6 +662,43 @@ def test_return_0_on_fail(self):
                 result = yaml.safe_load(handle)
 
 
+    def test_env_expansion(self):
+        """Ensure that config parser expands '${}' in env key"""
+
+        with tempfile.TemporaryDirectory() as tmp:
+            run_bc = Benchcomp({
+                "variants": {
+                    "env_set": {
+                        "config": {
+                            "command_line": 'echo "$__BENCHCOMP_ENV_VAR" > out',
+                            "directory": tmp,
+                            "env": {"__BENCHCOMP_ENV_VAR": "foo:${PATH}"}
+                        }
+                    },
+                },
+                "run": {
+                    "suites": {
+                        "suite_1": {
+                            "parser": {
+                                # The word 'bin' typically appears in $PATH, so
+                                # check that what was echoed contains 'bin'.
+                                "command": textwrap.dedent("""\
+                                    grep bin out && grep '^foo:' out && echo '{
+                                        "benchmarks": {},
+                                        "metrics": {}
+                                    }'
+                                    """)
+                            },
+                            "variants": ["env_set"]
+                        }
+                    }
+                },
+                "visualize": [],
+            })
+            run_bc()
+            self.assertEqual(run_bc.proc.returncode, 0, msg=run_bc.stderr)
+
+
     def test_env(self):
         """Ensure that benchcomp reads the 'env' key of variant config"""
 

tools/benchcomp/test/test_unit.py

@@ -0,0 +1,66 @@
+# Copyright Kani Contributors
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# Benchcomp regression testing suite. This suite uses Python's stdlib unittest
+# module, but nevertheless actually runs the binary rather than running unit
+# tests.
+
+import unittest
+import uuid
+
+import benchcomp.entry.run
+
+
+
+class TestEnvironmentUpdater(unittest.TestCase):
+    def test_environment_construction(self):
+        """Test that the default constructor reads the OS environment"""
+
+        update_environment = benchcomp.entry.run._EnvironmentUpdater()
+        environment = update_environment({})
+        self.assertIn("PATH", environment)
+
+
+    def test_placeholder_construction(self):
+        """Test that the placeholder constructor reads the placeholder"""
+
+        key, value = [str(uuid.uuid4()) for _ in range(2)]
+        update_environment = benchcomp.entry.run._EnvironmentUpdater({
+            key: value,
+        })
+        environment = update_environment({})
+        self.assertIn(key, environment)
+        self.assertEqual(environment[key], value)
+
+
+    def test_environment_update(self):
+        """Test that the environment is updated"""
+
+        key, value, update = [str(uuid.uuid4()) for _ in range(3)]
+        update_environment = benchcomp.entry.run._EnvironmentUpdater({
+            key: value,
+        })
+        environment = update_environment({
+            key: update
+        })
+        self.assertIn(key, environment)
+        self.assertEqual(environment[key], update)
+
+
+    def test_environment_update_variable(self):
+        """Test that the environment is updated"""
+
+        old_env = {
+            "key1": str(uuid.uuid4()),
+            "key2": str(uuid.uuid4()),
+        }
+
+        actual_update = "${key2}xxx${key1}"
+        expected_update = f"{old_env['key2']}xxx{old_env['key1']}"
+
+        update_environment = benchcomp.entry.run._EnvironmentUpdater(old_env)
+        environment = update_environment({
+            "key1": actual_update,
+        })
+        self.assertIn("key1", environment)
+        self.assertEqual(environment["key1"], expected_update)