add --with-env flag to publish command

glours · glours · commit 840288895e67 · 2025-01-29T15:45:00.000+01:00
this flag allow publishing env variables in the Compose OCI artifact

Signed-off-by: Guillaume Lours &lt;705411+glours@users.noreply.github.com&gt;
diff --git a/cmd/compose/publish.go b/cmd/compose/publish.go
@@ -29,6 +29,7 @@ type publishOptions struct {
 	*ProjectOptions
 	resolveImageDigests bool
 	ociVersion          string
+	withEnvironment     bool
 }
 
 func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command {
@@ -45,7 +46,9 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic
 	}
 	flags := cmd.Flags()
 	flags.BoolVar(&opts.resolveImageDigests, "resolve-image-digests", false, "Pin image tags to digests")
-	flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI Image/Artifact specification version (automatically determined by default)")
+	flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI image/artifact specification version (automatically determined by default)")
+	flags.BoolVar(&opts.withEnvironment, "with-env", false, "Include environment variables in the published OCI artifact")
+
 	return cmd
 }
 
@@ -58,5 +61,6 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service,
 	return backend.Publish(ctx, project, repository, api.PublishOptions{
 		ResolveImageDigests: opts.resolveImageDigests,
 		OCIVersion:          api.OCIVersion(opts.ociVersion),
+		WithEnvironment:     opts.withEnvironment,
 	})
 }
diff --git a/docs/reference/compose_alpha_publish.md b/docs/reference/compose_alpha_publish.md
@@ -8,8 +8,9 @@ Publish compose application
 | Name                      | Type     | Default | Description                                                                    |
 |:--------------------------|:---------|:--------|:-------------------------------------------------------------------------------|
 | `--dry-run`               | `bool`   |         | Execute command in dry run mode                                                |
-| `--oci-version`           | `string` |         | OCI Image/Artifact specification version (automatically determined by default) |
+| `--oci-version`           | `string` |         | OCI image/artifact specification version (automatically determined by default) |
 | `--resolve-image-digests` | `bool`   |         | Pin image tags to digests                                                      |
+| `--with-env`              | `bool`   |         | Include environment variables in the published OCI artifact                    |
 
 
 <!---MARKER_GEN_END-->
diff --git a/docs/reference/docker_compose_alpha_publish.yaml b/docs/reference/docker_compose_alpha_publish.yaml
@@ -8,7 +8,7 @@ options:
     - option: oci-version
       value_type: string
       description: |
-        OCI Image/Artifact specification version (automatically determined by default)
+        OCI image/artifact specification version (automatically determined by default)
       deprecated: false
       hidden: false
       experimental: false
@@ -25,6 +25,16 @@ options:
       experimentalcli: false
       kubernetes: false
       swarm: false
+    - option: with-env
+      value_type: bool
+      default_value: "false"
+      description: Include environment variables in the published OCI artifact
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
 inherited_options:
     - option: dry-run
       value_type: bool
diff --git a/internal/ocipush/push.go b/internal/ocipush/push.go
@@ -54,6 +54,8 @@ const (
 	// 	> an artifactType field, and tooling to work with artifacts should
 	//	> fallback to the config.mediaType value.
 	ComposeEmptyConfigMediaType = "application/vnd.docker.compose.config.empty.v1+json"
+	// ComposeEnvFileMediaType is the media type for each Env File layer in the image manifest.
+	ComposeEnvFileMediaType = "application/vnd.docker.compose.envfile"
 )
 
 // clientAuthStatusCodes are client (4xx) errors that are authentication
@@ -81,6 +83,18 @@ func DescriptorForComposeFile(path string, content []byte) v1.Descriptor {
 	}
 }
 
+func DescriptorForEnvFile(path string, content []byte) v1.Descriptor {
+	return v1.Descriptor{
+		MediaType: ComposeEnvFileMediaType,
+		Digest:    digest.FromString(string(content)),
+		Size:      int64(len(content)),
+		Annotations: map[string]string{
+			"com.docker.compose.version": api.ComposeVersion,
+			"com.docker.compose.envfile": filepath.Base(path),
+		},
+	}
+}
+
 func PushManifest(
 	ctx context.Context,
 	resolver *imagetools.Resolver,
diff --git a/pkg/api/api.go b/pkg/api/api.go
@@ -422,6 +422,7 @@ const (
 // PublishOptions group options of the Publish API
 type PublishOptions struct {
 	ResolveImageDigests bool
+	WithEnvironment     bool
 
 	OCIVersion OCIVersion
 }
diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go
@@ -18,6 +18,7 @@ package compose
 
 import (
 	"context"
+	"fmt"
 	"os"
 
 	"github.com/compose-spec/compose-go/v2/types"
@@ -35,7 +36,11 @@ func (s *composeService) Publish(ctx context.Context, project *types.Project, re
 }
 
 func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
-	err := s.Push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
+	err := preChecks(project, options)
+	if err != nil {
+		return err
+	}
+	err = s.Push(ctx, project, api.PushOptions{IgnoreFailures: true, ImageMandatory: true})
 	if err != nil {
 		return err
 	}
@@ -63,6 +68,10 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 		})
 	}
 
+	if options.WithEnvironment {
+		layers = append(layers, envFileLayers(project)...)
+	}
+
 	if options.ResolveImageDigests {
 		yaml, err := s.generateImageDigestsOverride(ctx, project)
 		if err != nil {
@@ -120,3 +129,49 @@ func (s *composeService) generateImageDigestsOverride(ctx context.Context, proje
 	}
 	return override.MarshalYAML()
 }
+
+func preChecks(project *types.Project, options api.PublishOptions) error {
+	if !options.WithEnvironment {
+		for _, service := range project.Services {
+			if len(service.EnvFiles) > 0 {
+				return fmt.Errorf("service %q has env_file declared. To avoid leaking sensitive data, "+
+					"you must either explicitly allow the sending of environment variables by using the --with-env flag,"+
+					" or remove sensitive data from your Compose configuration", service.Name)
+			}
+			if len(service.Environment) > 0 {
+				return fmt.Errorf("service %q has environment variable(s) declared. To avoid leaking sensitive data, "+
+					"you must either explicitly allow the sending of environment variables by using the --with-env flag,"+
+					" or remove sensitive data from your Compose configuration", service.Name)
+			}
+		}
+
+		for _, config := range project.Configs {
+			if config.Environment != "" {
+				return fmt.Errorf("config %q is declare as an environment variable. To avoid leaking sensitive data, "+
+					"you must either explicitly allow the sending of environment variables by using the --with-env flag,"+
+					" or remove sensitive data from your Compose configuration", config.Name)
+			}
+		}
+	}
+
+	return nil
+}
+
+func envFileLayers(project *types.Project) []ocipush.Pushable {
+	var layers []ocipush.Pushable
+	for _, service := range project.Services {
+		for _, envFile := range service.EnvFiles {
+			f, err := os.ReadFile(envFile.Path)
+			if err != nil {
+				// if we can't read the file, skip to the next one
+				continue
+			}
+			layerDescriptor := ocipush.DescriptorForEnvFile(envFile.Path, f)
+			layers = append(layers, ocipush.Pushable{
+				Descriptor: layerDescriptor,
+				Data:       f,
+			})
+		}
+	}
+	return layers
+}
diff --git a/pkg/e2e/fixtures/publish/compose-env-file.yml b/pkg/e2e/fixtures/publish/compose-env-file.yml
@@ -0,0 +1,7 @@
+services:
+  serviceA:
+    image: "alpine:3.12"
+    env_file:
+      - publish.env
+  serviceB:
+    image: "alpine:3.12"
diff --git a/pkg/e2e/fixtures/publish/compose-environment.yml b/pkg/e2e/fixtures/publish/compose-environment.yml
@@ -0,0 +1,7 @@
+services:
+  serviceA:
+    image: "alpine:3.12"
+    environment:
+        - "FOO=bar"
+  serviceB:
+    image: "alpine:3.12"
diff --git a/pkg/e2e/fixtures/publish/publish.env b/pkg/e2e/fixtures/publish/publish.env
@@ -0,0 +1 @@
+FOO=bar
diff --git a/pkg/e2e/publish_test.go b/pkg/e2e/publish_test.go
@@ -0,0 +1,56 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/icmd"
+)
+
+func TestPublishChecks(t *testing.T) {
+	c := NewParallelCLI(t)
+	const projectName = "compose-e2e-explicit-profiles"
+
+	t.Run("publish error environment", func(t *testing.T) {
+		res := c.RunDockerComposeCmdNoCheck(t, "-f", "./fixtures/publish/compose-environment.yml",
+			"-p", projectName, "alpha", "publish", "test/test")
+		res.Assert(t, icmd.Expected{ExitCode: 1, Err: `service "serviceA" has environment variable(s) declared. To avoid leaking sensitive data,`})
+	})
+
+	t.Run("publish error env_file", func(t *testing.T) {
+		res := c.RunDockerComposeCmdNoCheck(t, "-f", "./fixtures/publish/compose-env-file.yml",
+			"-p", projectName, "alpha", "publish", "test/test")
+		res.Assert(t, icmd.Expected{ExitCode: 1, Err: `service "serviceA" has env_file declared. To avoid leaking sensitive data,`})
+	})
+
+	t.Run("publish success environment", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/publish/compose-environment.yml",
+			"-p", projectName, "alpha", "publish", "test/test", "--with-env", "--dry-run")
+		assert.Assert(t, strings.Contains(res.Combined(), "test/test publishing"), res.Combined())
+		assert.Assert(t, strings.Contains(res.Combined(), "test/test published"), res.Combined())
+	})
+
+	t.Run("publish success env_file", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/publish/compose-env-file.yml",
+			"-p", projectName, "alpha", "publish", "test/test", "--with-env", "--dry-run")
+		assert.Assert(t, strings.Contains(res.Combined(), "test/test publishing"), res.Combined())
+		assert.Assert(t, strings.Contains(res.Combined(), "test/test published"), res.Combined())
+	})
+}
diff --git a/pkg/remote/oci.go b/pkg/remote/oci.go
@@ -154,17 +154,46 @@ func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, com
 		if err != nil {
 			return err
 		}
-		if i > 0 {
-			_, err = f.Write([]byte("\n---\n"))
-			if err != nil {
+
+		switch layer.MediaType {
+		case ocipush.ComposeYAMLMediaType:
+			if err := writeComposeFile(layer, i, f, content); err != nil {
 				return err
 			}
+		case ocipush.ComposeEnvFileMediaType:
+			if err := writeEnvFile(layer, local, content); err != nil {
+				return err
+			}
+		case ocipush.ComposeEmptyConfigMediaType:
 		}
-		_, err = f.Write(content)
+	}
+	return nil
+}
+
+func writeComposeFile(layer v1.Descriptor, i int, f *os.File, content []byte) error {
+	if _, ok := layer.Annotations["com.docker.compose.file"]; i > 0 && ok {
+		_, err := f.Write([]byte("\n---\n"))
 		if err != nil {
 			return err
 		}
 	}
+	_, err := f.Write(content)
+	return err
+}
+
+func writeEnvFile(layer v1.Descriptor, local string, content []byte) error {
+	envfilePath, ok := layer.Annotations["com.docker.compose.envfile"]
+	if !ok {
+		return fmt.Errorf("missing annotation com.docker.compose.envfile in layer %q", layer.Digest)
+	}
+	otherFile, err := os.Create(filepath.Join(local, envfilePath))
+	if err != nil {
+		return err
+	}
+	_, err = otherFile.Write(content)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ type publishOptions struct {`
`29`	`29`	`*ProjectOptions`
`30`	`30`	`resolveImageDigests bool`
`31`	`31`	`ociVersion string`
	`32`	`+ withEnvironment bool`
`32`	`33`	`}`
`33`	`34`
`34`	`35`	`func publishCommand(p ProjectOptions, dockerCli command.Cli, backend api.Service) cobra.Command {`
`@@ -45,7 +46,9 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic`
`45`	`46`	`}`
`46`	`47`	`flags := cmd.Flags()`
`47`	`48`	`flags.BoolVar(&opts.resolveImageDigests, "resolve-image-digests", false, "Pin image tags to digests")`
`48`		`- flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI Image/Artifact specification version (automatically determined by default)")`
	`49`	`+ flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI image/artifact specification version (automatically determined by default)")`
	`50`	`+ flags.BoolVar(&opts.withEnvironment, "with-env", false, "Include environment variables in the published OCI artifact")`
	`51`	`+`
`49`	`52`	`return cmd`
`50`	`53`	`}`
`51`	`54`
`@@ -58,5 +61,6 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service,`
`58`	`61`	`return backend.Publish(ctx, project, repository, api.PublishOptions{`
`59`	`62`	`ResolveImageDigests: opts.resolveImageDigests,`
`60`	`63`	`OCIVersion: api.OCIVersion(opts.ociVersion),`
	`64`	`+ WithEnvironment: opts.withEnvironment,`
`61`	`65`	`})`
`62`	`66`	`}`
Original file line number	Diff line number	Diff line change
`@@ -422,6 +422,7 @@ const (`
`422`	`422`	`// PublishOptions group options of the Publish API`
`423`	`423`	`type PublishOptions struct {`
`424`	`424`	`ResolveImageDigests bool`
	`425`	`+ WithEnvironment bool`
`425`	`426`
`426`	`427`	`OCIVersion OCIVersion`
`427`	`428`	`}`