From ea4c6174fa8850bf21c2309290853ccc4a0a0d57 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Thu, 9 Jul 2020 18:59:52 -0700 Subject: [PATCH 1/9] parser update. TODO: make test for parser --- src/popper/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/popper/parser.py b/src/popper/parser.py index 7c00a539a..656dfc59f 100644 --- a/src/popper/parser.py +++ b/src/popper/parser.py @@ -35,6 +35,7 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, + "options": {"type": "seq", "sequence" : [{"type": "any"}]} }, } ], From 75500442474aa36190fab6e3d0ed8507ddbda098 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Sun, 12 Jul 2020 17:01:57 -0700 Subject: [PATCH 2/9] parser working --- src/popper/parser.py | 3 ++- src/test/test_parser.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/popper/parser.py b/src/popper/parser.py index 656dfc59f..5fdafadec 100644 --- a/src/popper/parser.py +++ b/src/popper/parser.py @@ -35,7 +35,7 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, - "options": {"type": "seq", "sequence" : [{"type": "any"}]} + "container_config": {"type": "map", "allowempty": True} }, } ], @@ -49,6 +49,7 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, + }, }, }, diff --git a/src/test/test_parser.py b/src/test/test_parser.py index a40e25907..66260eb5a 100644 --- a/src/test/test_parser.py +++ b/src/test/test_parser.py @@ -37,10 +37,11 @@ def test_new_workflow(self): "env": {"EN": "EE"}, "secrets": ["S"], "dir": "/path/to/", + "container_config": {"name": "spam"} }, {"uses": "bar", "runs": ["a", "b"], "args": ["c"], "skip_pull": True}, ], - "options": {"env": {"FOO": "bar"}, "secrets": ["Z"]}, + "options": {"env": {"FOO": "bar"}, "secrets": ["Z"], }, } wf = WorkflowParser.parse(wf_data=wf_data) @@ -50,6 +51,7 @@ def test_new_workflow(self): self.assertEqual(("Z", "S"), step.secrets) self.assertEqual({"EN": "EE", "FOO": "bar"}, step.env) self.assertEqual("/path/to/", step.dir) + self.assertEqual("spam", step.container_options.name) self.assertTrue(not step.runs) self.assertTrue(not step.args) self.assertFalse(step.skip_pull) From 36774199810fdd53eb720f51c28f8dea76323c19 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Sun, 12 Jul 2020 22:21:44 -0700 Subject: [PATCH 3/9] docker arguments update working --- src/popper/parser.py | 3 +-- src/popper/runner_host.py | 2 +- src/test/test_parser.py | 4 ++-- src/test/test_runner_host.py | 9 ++++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/popper/parser.py b/src/popper/parser.py index 5fdafadec..d13a9eb68 100644 --- a/src/popper/parser.py +++ b/src/popper/parser.py @@ -35,7 +35,7 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, - "container_config": {"type": "map", "allowempty": True} + "container_config": {"type": "map", "allowempty": True}, }, } ], @@ -49,7 +49,6 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, - }, }, }, diff --git a/src/popper/runner_host.py b/src/popper/runner_host.py index 6dc98f1d5..9b4de6529 100644 --- a/src/popper/runner_host.py +++ b/src/popper/runner_host.py @@ -257,7 +257,7 @@ def _get_container_kwargs(self, step, img, name): } self._update_with_engine_config(args) - + args.update(step.container_config) log.debug(f"container args: {pu.prettystr(args)}\n") return args diff --git a/src/test/test_parser.py b/src/test/test_parser.py index 66260eb5a..b4dee20af 100644 --- a/src/test/test_parser.py +++ b/src/test/test_parser.py @@ -37,11 +37,11 @@ def test_new_workflow(self): "env": {"EN": "EE"}, "secrets": ["S"], "dir": "/path/to/", - "container_config": {"name": "spam"} + "container_config": {"name": "spam"}, }, {"uses": "bar", "runs": ["a", "b"], "args": ["c"], "skip_pull": True}, ], - "options": {"env": {"FOO": "bar"}, "secrets": ["Z"], }, + "options": {"env": {"FOO": "bar"}, "secrets": ["Z"],}, } wf = WorkflowParser.parse(wf_data=wf_data) diff --git a/src/test/test_runner_host.py b/src/test/test_runner_host.py index 3f2fe78b3..5f2d409ac 100644 --- a/src/test/test_runner_host.py +++ b/src/test/test_runner_host.py @@ -159,6 +159,7 @@ def test_get_container_kwargs(self): "args": ["ls"], "id": "one", "dir": "/tmp/", + "container_config": {"privileged": False, "ports": {'8888/tcp': 8888}}, }, default_box=True, ) @@ -182,7 +183,7 @@ def test_get_container_kwargs(self): with DockerRunner(init_docker_client=False, config=config) as dr: args = dr._get_container_kwargs(step, "alpine:3.9", "container_a") - + self.assertEqual( args, { @@ -200,9 +201,10 @@ def test_get_container_kwargs(self): "detach": True, "stdin_open": False, "tty": False, - "privileged": True, + "privileged": False, "hostname": "popper.local", "domainname": "www.example.org", + "ports": {'8888/tcp': 8888}, }, ) @@ -231,9 +233,10 @@ def test_get_container_kwargs(self): "detach": False, "stdin_open": True, "tty": True, - "privileged": True, + "privileged": False, "hostname": "popper.local", "domainname": "www.example.org", + "ports": {'8888/tcp': 8888}, }, ) From c81dc3a551b887b9bb3b17ae0894f5f13dfb9912 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Mon, 13 Jul 2020 02:21:51 -0700 Subject: [PATCH 4/9] renamed container_config to options --- src/popper/parser.py | 2 +- src/test/test_parser.py | 2 +- src/test/test_runner_host.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/popper/parser.py b/src/popper/parser.py index d13a9eb68..05ed62ed9 100644 --- a/src/popper/parser.py +++ b/src/popper/parser.py @@ -35,7 +35,7 @@ class WorkflowParser(object): "matching-rule": "any", "mapping": {"regex;(.+)": {"type": "str"}}, }, - "container_config": {"type": "map", "allowempty": True}, + "options": {"type": "map", "allowempty": True}, }, } ], diff --git a/src/test/test_parser.py b/src/test/test_parser.py index b4dee20af..6a431d7fc 100644 --- a/src/test/test_parser.py +++ b/src/test/test_parser.py @@ -37,7 +37,7 @@ def test_new_workflow(self): "env": {"EN": "EE"}, "secrets": ["S"], "dir": "/path/to/", - "container_config": {"name": "spam"}, + "options": {"name": "spam"}, }, {"uses": "bar", "runs": ["a", "b"], "args": ["c"], "skip_pull": True}, ], diff --git a/src/test/test_runner_host.py b/src/test/test_runner_host.py index 5f2d409ac..5fe5b2479 100644 --- a/src/test/test_runner_host.py +++ b/src/test/test_runner_host.py @@ -159,7 +159,7 @@ def test_get_container_kwargs(self): "args": ["ls"], "id": "one", "dir": "/tmp/", - "container_config": {"privileged": False, "ports": {'8888/tcp': 8888}}, + "options": {"privileged": False, "ports": {'8888/tcp': 8888}}, }, default_box=True, ) @@ -184,6 +184,8 @@ def test_get_container_kwargs(self): with DockerRunner(init_docker_client=False, config=config) as dr: args = dr._get_container_kwargs(step, "alpine:3.9", "container_a") + print() + self.assertEqual( args, { From feef1617002830dec419447ee4b9ed5041d9edbb Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Mon, 13 Jul 2020 02:29:45 -0700 Subject: [PATCH 5/9] errors fixed --- src/popper/runner_host.py | 2 +- src/test/test_parser.py | 2 +- src/test/test_runner_host.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/popper/runner_host.py b/src/popper/runner_host.py index 9b4de6529..a709d7894 100644 --- a/src/popper/runner_host.py +++ b/src/popper/runner_host.py @@ -257,7 +257,7 @@ def _get_container_kwargs(self, step, img, name): } self._update_with_engine_config(args) - args.update(step.container_config) + args.update(step.options) log.debug(f"container args: {pu.prettystr(args)}\n") return args diff --git a/src/test/test_parser.py b/src/test/test_parser.py index 6a431d7fc..9eb598233 100644 --- a/src/test/test_parser.py +++ b/src/test/test_parser.py @@ -51,7 +51,7 @@ def test_new_workflow(self): self.assertEqual(("Z", "S"), step.secrets) self.assertEqual({"EN": "EE", "FOO": "bar"}, step.env) self.assertEqual("/path/to/", step.dir) - self.assertEqual("spam", step.container_options.name) + self.assertEqual("spam", step.options.name) self.assertTrue(not step.runs) self.assertTrue(not step.args) self.assertFalse(step.skip_pull) diff --git a/src/test/test_runner_host.py b/src/test/test_runner_host.py index 5fe5b2479..f869f2e00 100644 --- a/src/test/test_runner_host.py +++ b/src/test/test_runner_host.py @@ -159,7 +159,7 @@ def test_get_container_kwargs(self): "args": ["ls"], "id": "one", "dir": "/tmp/", - "options": {"privileged": False, "ports": {'8888/tcp': 8888}}, + "options": {"ports": {"8888/tcp": 8888}}, }, default_box=True, ) @@ -183,7 +183,7 @@ def test_get_container_kwargs(self): with DockerRunner(init_docker_client=False, config=config) as dr: args = dr._get_container_kwargs(step, "alpine:3.9", "container_a") - + print() self.assertEqual( @@ -203,10 +203,10 @@ def test_get_container_kwargs(self): "detach": True, "stdin_open": False, "tty": False, - "privileged": False, + "privileged": True, "hostname": "popper.local", "domainname": "www.example.org", - "ports": {'8888/tcp': 8888}, + "ports": {"8888/tcp": 8888}, }, ) @@ -235,10 +235,10 @@ def test_get_container_kwargs(self): "detach": False, "stdin_open": True, "tty": True, - "privileged": False, + "privileged": True, "hostname": "popper.local", "domainname": "www.example.org", - "ports": {'8888/tcp': 8888}, + "ports": {"8888/tcp": 8888}, }, ) From ebc1708b3a81ab8be4d294b373bc4b5dd7d14fb8 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Mon, 13 Jul 2020 08:58:32 -0700 Subject: [PATCH 6/9] removed debugging print statement --- src/test/test_runner_host.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/test_runner_host.py b/src/test/test_runner_host.py index f869f2e00..c066bdb59 100644 --- a/src/test/test_runner_host.py +++ b/src/test/test_runner_host.py @@ -184,8 +184,6 @@ def test_get_container_kwargs(self): with DockerRunner(init_docker_client=False, config=config) as dr: args = dr._get_container_kwargs(step, "alpine:3.9", "container_a") - print() - self.assertEqual( args, { From e97089e8e0b33a67e6652af3d1937012d3950ae3 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Tue, 14 Jul 2020 19:25:39 -0700 Subject: [PATCH 7/9] documentation for container options for the Docker runner --- docs/sections/cn_workflows.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/sections/cn_workflows.md b/docs/sections/cn_workflows.md index d42301b4d..1dbca729b 100644 --- a/docs/sections/cn_workflows.md +++ b/docs/sections/cn_workflows.md @@ -43,7 +43,9 @@ step. All attributes are optional with the exception of the `uses` attribute. | `env` | **optional** The environment variables to set inside the container's runtime environment. If
you need to pass environment variables into a step, make sure it runs a command
shell to perform variable substitution. For example, if your `runs` attribute is
set to `["sh", "-c"]`, the value of `args` will be passed to `sh -c` and
executed in a command shell. Alternatively, if your `Dockerfile` uses an
`ENTRYPOINT` to run the same command (`"sh -c"`), `args` will execute in a
command shell as well. See [`ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) for more details. | | `secrets` | **optional** Specifies the names of the secret variables to set in the runtime environment
which the container can access as an environment variable. For example,
`secrets: ["SECRET1", "SECRET2"]`. | | `skip_pull` | **optional** Assume that the given container image already exist and skip pulling it. | -| `dir` | **optional** Specifies the working directory for a step. By default, the directory is always `/workspace` if another one is not defined. | +| `dir` | **opftional** Specifies the working directory for a step. By default, the directory is always `/workspace` if another one is not defined. | +| `options` | **optional** Container configuration options. For instance: `options: {ports: {8888:8888}, interactive: True, tty: True}`. Currently only supported for the docker runtime. See the parameters of `client.containers.runs()` in the [Docker Python SDK](https://docker-py.readthedocs.io/en/stable/containers.html?highlight=inspect) for the full list of options +| ### Referencing images in a step @@ -398,6 +400,11 @@ question (see [here][engconf] for more). [engconf]: ./cli_features#customizing-container-engine-behavior + +Alternatively, to restrict a configuration to a specific step in a workflow, set the desired parameters in the step's `options` +**Note**: this is currently only supported for the Docker runtime + + ## Resource Managers Popper can execute steps in a workflow through other resource managers From 1aa9d935b1661ff38ad430812d48181948086bbf Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Wed, 15 Jul 2020 08:17:29 -0700 Subject: [PATCH 8/9] title typo --- docs/sections/cli_features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/cli_features.md b/docs/sections/cli_features.md index 7d88460bb..494b5a2cb 100644 --- a/docs/sections/cli_features.md +++ b/docs/sections/cli_features.md @@ -1,4 +1,4 @@ -# CLI feautures +# CLI features ## New workflow initialization From d595d79af0b3ef4ae2b7f58061b1dec1e2d46837 Mon Sep 17 00:00:00 2001 From: Anders Poirel Date: Wed, 15 Jul 2020 09:06:07 -0700 Subject: [PATCH 9/9] Update cn_workflows.md --- docs/sections/cn_workflows.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sections/cn_workflows.md b/docs/sections/cn_workflows.md index 1dbca729b..1d6d8da4a 100644 --- a/docs/sections/cn_workflows.md +++ b/docs/sections/cn_workflows.md @@ -44,8 +44,7 @@ step. All attributes are optional with the exception of the `uses` attribute. | `secrets` | **optional** Specifies the names of the secret variables to set in the runtime environment
which the container can access as an environment variable. For example,
`secrets: ["SECRET1", "SECRET2"]`. | | `skip_pull` | **optional** Assume that the given container image already exist and skip pulling it. | | `dir` | **opftional** Specifies the working directory for a step. By default, the directory is always `/workspace` if another one is not defined. | -| `options` | **optional** Container configuration options. For instance: `options: {ports: {8888:8888}, interactive: True, tty: True}`. Currently only supported for the docker runtime. See the parameters of `client.containers.runs()` in the [Docker Python SDK](https://docker-py.readthedocs.io/en/stable/containers.html?highlight=inspect) for the full list of options -| +| `options` | **optional** Container configuration options. For instance: `options: {ports: {8888:8888}, interactive: True, tty: True}`. Currently only supported for the docker runtime. See the parameters of `client.containers.runs()` in the [Docker Python SDK](https://docker-py.readthedocs.io/en/stable/containers.html?highlight=inspect) for the full list of options | ### Referencing images in a step