From bf735931d00e860cf83f6074d7df47cf52ffbf07 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Tue, 13 May 2025 17:59:48 -0600 Subject: [PATCH 01/43] partial sdk client, more tests --- pkg/docker/api_client.go | 214 +++++++++++ pkg/docker/credentials.go | 33 ++ pkg/docker/docker_client_test.go | 353 +++++++++++------- .../registry_container.go | 14 + 4 files changed, 488 insertions(+), 126 deletions(-) create mode 100644 pkg/docker/api_client.go create mode 100644 pkg/docker/credentials.go diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go new file mode 100644 index 0000000000..7b4346e988 --- /dev/null +++ b/pkg/docker/api_client.go @@ -0,0 +1,214 @@ +package docker + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/client" + dc "github.com/docker/docker/client" + + "github.com/replicate/go/types/ptr" + + "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" +) + +func NewAPIClient(ctx context.Context) (*apiClient, error) { + // clicfg := cliconfig.FromContext(ctx) + + client, err := dc.NewClientWithOpts(dc.FromEnv, dc.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("error creating docker client: %w", err) + } + + if _, err := client.Ping(ctx); err != nil { + return nil, fmt.Errorf("error pinging docker daemon: %w", err) + } + + authConfig := make(map[string]registry.AuthConfig) + userInfo, err := loadUserInformation(ctx, "r8.im") + if err != nil { + return nil, fmt.Errorf("error loading user information: %w", err) + } + authConfig["r8.im"] = registry.AuthConfig{ + Username: userInfo.Username, + Password: userInfo.Token, + } + + return &apiClient{client, authConfig}, nil +} + +type apiClient struct { + client *dc.Client + authConfig map[string]registry.AuthConfig +} + +func (c *apiClient) Pull(ctx context.Context, imageRef string, force bool) (*image.InspectResponse, error) { + console.Debugf("=== APIClient.Pull %s force:%t", imageRef, force) + + if !force { + inspect, err := c.Inspect(ctx, imageRef) + if err == nil { + return inspect, nil + } else if !command.IsNotFoundError(err) { + // Log a warning if inspect fails for any reason other than not found. + // It's likely that pull will fail as well, but it's better to return that error + // so the caller can handle it appropriately than to fail silently here. + console.Warnf("failed to inspect image before pulling %q: %s", imageRef, err) + } + } + + output, err := c.client.ImagePull(ctx, imageRef, image.PullOptions{ + // force image to linux/amd64 to match production + Platform: "linux/amd64", + }) + if err != nil { + if client.IsErrNotFound(err) { + return nil, &command.NotFoundError{Ref: imageRef, Object: "image"} + } + return nil, fmt.Errorf("failed to pull image %q: %w", imageRef, err) + } + defer output.Close() + io.Copy(os.Stderr, output) + + // pull succeeded, inspect the image again and return + inspect, err := c.Inspect(ctx, imageRef) + if err != nil { + return nil, fmt.Errorf("failed to inspect image after pulling %q: %w", imageRef, err) + } + return inspect, nil +} + +func (c *apiClient) ContainerStop(ctx context.Context, containerID string) error { + console.Debugf("=== APIClient.ContainerStop %s", containerID) + + err := c.client.ContainerStop(ctx, containerID, container.StopOptions{ + Timeout: ptr.To(3), + }) + if err != nil { + if client.IsErrNotFound(err) { + return &command.NotFoundError{Ref: containerID, Object: "container"} + } + return fmt.Errorf("failed to stop container %q: %w", containerID, err) + } + return nil +} + +func (c *apiClient) ContainerInspect(ctx context.Context, containerID string) (*container.InspectResponse, error) { + console.Debugf("=== APIClient.ContainerInspect %s", containerID) + + resp, err := c.client.ContainerInspect(ctx, containerID) + if err != nil { + if client.IsErrNotFound(err) { + return nil, &command.NotFoundError{Ref: containerID, Object: "container"} + } + return nil, fmt.Errorf("failed to inspect container %q: %w", containerID, err) + } + return &resp, nil +} + +func (c *apiClient) ContainerLogs(ctx context.Context, containerID string, w io.Writer) error { + console.Debugf("=== APIClient.ContainerLogs %s", containerID) + + logs, err := c.client.ContainerLogs(ctx, containerID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + Follow: true, + }) + if err != nil { + if client.IsErrNotFound(err) { + return &command.NotFoundError{Ref: containerID, Object: "container"} + } + return fmt.Errorf("failed to get container logs for %q: %w", containerID, err) + } + defer logs.Close() + // Docker adds a header to each log line. The header is 8 bytes: + // - First byte is the stream type (1 for stdout, 2 for stderr) + // - Next 3 bytes are reserved + // - Last 4 bytes are the size of the message + // We want to strip the header and prefix each line with the stream type. Maybe... + // the CLI doesn't do any of this, so we might not need to do anything fancy. /shrug + scanner := bufio.NewScanner(logs) + for scanner.Scan() { + line := scanner.Text() + if len(line) < 8 { + continue + } + stream := line[0] + switch stream { + case '\x01': + fmt.Fprintln(w, "[stdout]", line[8:]) + case '\x02': + fmt.Fprintln(w, "[stderr]", line[8:]) + } + } + return nil +} + +func (c *apiClient) Push(ctx context.Context, ref string) error { + panic("not implemented") +} + +func (c *apiClient) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { + console.Debugf("=== APIClient.LoadUserInformation %s", registryHost) + panic("not implemented") +} + +func (c *apiClient) CreateTarFile(ctx context.Context, ref string, tmpDir string, tarFile string, folder string) (string, error) { + panic("not implemented") +} + +func (c *apiClient) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) { + panic("not implemented") +} + +func (c *apiClient) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) { + // TODO[md]: platform requires engine 1.49+, and it's not widly available as of 2025-05. + // platform := ocispec.Platform{OS: "linux", Architecture: "amd64"} + // client.ImageInspectWithPlatform(&platform), + inspect, err := c.client.ImageInspect(ctx, ref) + + if err != nil { + if client.IsErrNotFound(err) { + return nil, &command.NotFoundError{Ref: ref, Object: "image"} + } + return nil, fmt.Errorf("error inspecting image: %w", err) + } + + return &inspect, nil +} + +func (c *apiClient) ImageExists(ctx context.Context, ref string) (bool, error) { + console.Debugf("=== APIClient.ImageExists %s", ref) + + _, err := c.Inspect(ctx, ref) + if err != nil { + if command.IsNotFoundError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOptions) error { + panic("not implemented") +} + +func (c *apiClient) Run(ctx context.Context, options command.RunOptions) error { + panic("not implemented") +} + +func (c *apiClient) ContainerStart(ctx context.Context, options command.RunOptions) (string, error) { + panic("not implemented") +} + +func (c *apiClient) ContainerRemove(ctx context.Context, containerID string) error { + panic("not implemented") +} diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go new file mode 100644 index 0000000000..bf4b8690c0 --- /dev/null +++ b/pkg/docker/credentials.go @@ -0,0 +1,33 @@ +package docker + +import ( + "context" + "os" + + "github.com/docker/cli/cli/config" + + "github.com/replicate/cog/pkg/docker/command" +) + +func loadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { + conf := config.LoadDefaultConfigFile(os.Stderr) + credsStore := conf.CredentialsStore + if credsStore == "" { + authConf, err := loadAuthFromConfig(conf, registryHost) + if err != nil { + return nil, err + } + return &command.UserInfo{ + Token: authConf.Password, + Username: authConf.Username, + }, nil + } + credsHelper, err := loadAuthFromCredentialsStore(ctx, credsStore, registryHost) + if err != nil { + return nil, err + } + return &command.UserInfo{ + Token: credsHelper.Secret, + Username: credsHelper.Username, + }, nil +} diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index d5abbc919a..85530da572 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -1,12 +1,17 @@ package docker import ( + "bytes" + "fmt" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/docker/dockertest" @@ -18,163 +23,259 @@ func TestDockerClient(t *testing.T) { t.Skip("skipping docker client tests in short mode") } - suite := &DockerClientSuite{ - dockerHelper: dockertest.NewHelperClient(t), - dockerClient: NewDockerCommand(), - } - - t.Run("ImageInspect", suite.runImageInspectTests) - t.Run("Pull", suite.runPullTests) - t.Run("ContainerStop", suite.runContainerStopTests) + client := NewDockerCommand() + runDockerClientTests(t, client) } -type DockerClientSuite struct { - dockerHelper *dockertest.HelperClient - dockerClient command.Command -} +func TestDockerAPIClient(t *testing.T) { + if testing.Short() { + t.Skip("skipping docker client tests in short mode") + } -func (s *DockerClientSuite) assertImageExists(t *testing.T, imageRef string) { - inspect, err := s.dockerClient.Inspect(t.Context(), imageRef) - assert.NoError(t, err, "Failed to inspect image %q", imageRef) - assert.NotNil(t, inspect, "Image should exist") + apiClient, err := NewAPIClient(t.Context()) + require.NoError(t, err, "Failed to create docker api client") + runDockerClientTests(t, apiClient) } -func (s *DockerClientSuite) assertNoImageExists(t *testing.T, imageRef string) { - inspect, err := s.dockerClient.Inspect(t.Context(), imageRef) - assert.ErrorIs(t, err, &command.NotFoundError{}, "Image should not exist") - assert.Nil(t, inspect, "Image should not exist") -} - -func (s *DockerClientSuite) runImageInspectTests(t *testing.T) { - t.Run("ExistingLocalImage", func(t *testing.T) { - t.Parallel() - - image := "docker.io/library/busybox:latest" - - s.dockerHelper.MustPullImage(t, image) +func runDockerClientTests(t *testing.T, dockerClient command.Command) { + dockerHelper := dockertest.NewHelperClient(t) + testRegistry := registry_testhelpers.StartTestRegistry(t) - expectedImage := s.dockerHelper.InspectImage(t, image) - resp, err := s.dockerClient.Inspect(t.Context(), image) - require.NoError(t, err, "Failed to inspect image %q", image) - assert.Equal(t, expectedImage.ID, resp.ID) - }) + dockerHelper.CleanupImages(t) - t.Run("MissingLocalImage", func(t *testing.T) { + t.Run("ImageInspect", func(t *testing.T) { t.Parallel() - image := "not-a-valid-image" - _, err := s.dockerClient.Inspect(t.Context(), image) - assert.ErrorIs(t, err, &command.NotFoundError{}) - assert.ErrorContains(t, err, "image not found") - }) -} - -func (s *DockerClientSuite) runPullTests(t *testing.T) { - testRegistry := registry_testhelpers.StartTestRegistry(t) + t.Run("ExistingLocalImage", func(t *testing.T) { + t.Parallel() - // TODO[md]: add tests for the following permutations: - // - remote reference exists/not exists - // - local reference exists/not exists - // - force pull true/false + imageRef := imageRefForTest(t) + dockerHelper.LoadImageFixture(t, "alpine", imageRef) - t.Run("RemoteImageExists", func(t *testing.T) { - imageRef := testRegistry.ImageRefForTest(t, "") + expectedImage := dockerHelper.InspectImage(t, imageRef) + resp, err := dockerClient.Inspect(t.Context(), imageRef) + require.NoError(t, err, "Failed to inspect image %q", imageRef) + assert.Equal(t, expectedImage.ID, resp.ID) + }) - s.dockerHelper.LoadImageFixture(t, "alpine", imageRef) - s.dockerHelper.MustPushImage(t, imageRef) - s.dockerHelper.MustDeleteImage(t, imageRef) + t.Run("MissingLocalImage", func(t *testing.T) { + t.Parallel() - s.assertNoImageExists(t, imageRef) + image := "not-a-valid-image" + _, err := dockerClient.Inspect(t.Context(), image) + assert.ErrorIs(t, err, &command.NotFoundError{}) + assert.ErrorContains(t, err, "image not found") + }) + }) - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - require.NoError(t, err, "Failed to pull image %q", imageRef) - s.dockerHelper.CleanupImage(t, imageRef) + t.Run("Pull", func(t *testing.T) { + t.Parallel() - s.assertImageExists(t, imageRef) - expectedResp := s.dockerHelper.InspectImage(t, imageRef) - // TODO[md]: we should check that the responsees are actually equal beyond the IDs. but atm - // the CLI and api are slightly different. The CLI leaves the descriptor field nil while the - // API response is populated. These should be identical on the new client, so we can change to EqualValues - assert.Equal(t, expectedResp.ID, resp.ID, "inspect response should match expected") + // TODO[md]: add tests for the following permutations: + // - remote reference exists/not exists + // - local reference exists/not exists + // - force pull true/false + + t.Run("RemoteImageExists", func(t *testing.T) { + t.Parallel() + repo := testRegistry.CloneRepoForTest(t, "alpine") + imageRef := repo + ":latest" + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + require.NoError(t, err, "Failed to pull image %q", imageRef) + dockerHelper.CleanupImage(t, imageRef) + + assertImageExists(t, dockerClient, imageRef) + expectedResp := dockerHelper.InspectImage(t, imageRef) + // TODO[md]: we should check that the responsees are actually equal beyond the IDs. but atm + // the CLI and api are slightly different. The CLI leaves the descriptor field nil while the + // API response is populated. These should be identical on the new client, so we can change to EqualValues + assert.Equal(t, expectedResp.ID, resp.ID, "inspect response should match expected") + }) + + t.Run("RemoteReferenceNotFound", func(t *testing.T) { + t.Parallel() + imageRef := testRegistry.ImageRefForTest(t, "") + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry + // so we handle other failure cases, like failed auth, unknown tag, and unknown repo + require.Error(t, err, "Failed to pull image %q", imageRef) + assert.ErrorIs(t, err, &command.NotFoundError{Object: "manifest", Ref: imageRef}) + assert.Nil(t, resp, "inspect response should be nil") + }) + + t.Run("InvalidAuth", func(t *testing.T) { + t.Skip("skip auth tests until we're using the docker engine since we can't set auth on the host without side effects") + imageRef := testRegistry.ImageRefForTest(t, "") + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry + // so we handle other failure cases, like failed auth, unknown tag, and unknown repo + require.Error(t, err, "Failed to pull image %q", imageRef) + assert.ErrorContains(t, err, "failed to resolve reference") + assert.Nil(t, resp, "inspect response should be nil") + }) }) - t.Run("RemoteReferenceNotFound", func(t *testing.T) { - imageRef := testRegistry.ImageRefForTest(t, "") - - s.assertNoImageExists(t, imageRef) + t.Run("ContainerStop", func(t *testing.T) { + t.Parallel() - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry - // so we handle other failure cases, like failed auth, unknown tag, and unknown repo - require.Error(t, err, "Failed to pull image %q", imageRef) - assert.ErrorIs(t, err, &command.NotFoundError{Object: "manifest", Ref: imageRef}) - assert.Nil(t, resp, "inspect response should be nil") + t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sleep", "5000"), + ) + defer dockerHelper.CleanupImages(t) + defer testcontainers.CleanupContainer(t, container) + require.NoError(t, err, "Failed to run container") + + err = dockerClient.ContainerStop(t.Context(), container.ID) + require.NoError(t, err, "Failed to stop container %q", container.ID) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + }) + + t.Run("ContainerExistsAndIsNotRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.GenericContainer(t.Context(), + testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: testRegistry.ImageRef("alpine:latest"), + Cmd: []string{"sleep", "5000"}, + }, + Started: false, + }, + ) + defer testcontainers.CleanupContainer(t, container) + containerID := container.GetContainerID() + require.NoError(t, err, "Failed to create container") + + err = dockerClient.ContainerStop(t.Context(), containerID) + require.NoError(t, err, "Failed to stop container %q", containerID) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + }) + + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() + + err := dockerClient.ContainerStop(t.Context(), "containerid-that-does-not-exist") + require.ErrorIs(t, err, &command.NotFoundError{}) + require.ErrorContains(t, err, "container not found") + }) }) - t.Run("InvalidAuth", func(t *testing.T) { - t.Skip("skip auth tests until we're using the docker engine since we can't set auth on the host without side effects") - imageRef := testRegistry.ImageRefForTest(t, "") + t.Run("ContainerInspect", func(t *testing.T) { + t.Parallel() - s.assertNoImageExists(t, imageRef) + t.Run("ContainerExists", func(t *testing.T) { + t.Parallel() - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry - // so we handle other failure cases, like failed auth, unknown tag, and unknown repo - require.Error(t, err, "Failed to pull image %q", imageRef) - assert.ErrorContains(t, err, "failed to resolve reference") - assert.Nil(t, resp, "inspect response should be nil") - }) -} + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sleep", "5000"), + ) + defer testcontainers.CleanupContainer(t, container) + require.NoError(t, err, "Failed to run container") -func (s *DockerClientSuite) runContainerStopTests(t *testing.T) { - t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { - t.Parallel() + expected, err := container.Inspect(t.Context()) + require.NoError(t, err, "Failed to inspect container for expected response") - container, err := testcontainers.Run( - t.Context(), - "docker.io/library/busybox:latest", - testcontainers.WithCmd("sleep", "5000"), - ) - defer testcontainers.CleanupContainer(t, container) - require.NoError(t, err, "Failed to run container") + resp, err := dockerClient.ContainerInspect(t.Context(), container.ID) + require.NoError(t, err, "Failed to inspect container") + require.Equal(t, expected, resp) + }) - err = s.dockerClient.ContainerStop(t.Context(), container.ID) - require.NoError(t, err, "Failed to stop container %q", container.ID) + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() - state, err := container.State(t.Context()) - require.NoError(t, err, "Failed to get container state") - assert.Equal(t, state.Running, false) + _, err := dockerClient.ContainerInspect(t.Context(), "containerid-that-does-not-exist") + require.ErrorIs(t, err, &command.NotFoundError{}) + }) }) - t.Run("ContainerExistsAndIsNotRunning", func(t *testing.T) { + t.Run("ContainerLogs", func(t *testing.T) { t.Parallel() - container, err := testcontainers.GenericContainer(t.Context(), - testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "docker.io/library/busybox:latest", - Cmd: []string{"sleep", "5000"}, - }, - Started: false, - }, - ) - defer testcontainers.CleanupContainer(t, container) - containerID := container.GetContainerID() - require.NoError(t, err, "Failed to create container") - - err = s.dockerClient.ContainerStop(t.Context(), containerID) - require.NoError(t, err, "Failed to stop container %q", containerID) - - state, err := container.State(t.Context()) - require.NoError(t, err, "Failed to get container state") - assert.Equal(t, state.Running, false) + t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + // print "line $i" N times then exit, where $i is the line number + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"line $i\"; sleep 1; done"), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + + assert.Equal(t, "[stdout] line 1\n[stdout] line 2\n[stdout] line 3\n[stdout] line 4\n[stdout] line 5\n", buf.String()) + }) + + t.Run("ContainerAlreadyStopped", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 3); do echo \"line $i\"; sleep 0.1; done"), + testcontainers.WithWaitStrategy(wait.ForExit()), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + + assert.Equal(t, "[stdout] line 1\n[stdout] line 2\n[stdout] line 3\n", buf.String()) + }) + + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() + + err := dockerClient.ContainerLogs(t.Context(), "containerid-that-does-not-exist", &bytes.Buffer{}) + require.ErrorIs(t, err, &command.NotFoundError{}) + }) }) +} - t.Run("ContainerDoesNotExist", func(t *testing.T) { - t.Parallel() +func imageRefForTest(t *testing.T) string { + return fmt.Sprintf("%s:test-%d", strings.ToLower(t.Name()), time.Now().Unix()) +} - err := s.dockerClient.ContainerStop(t.Context(), "containerid-that-does-not-exist") - require.ErrorIs(t, err, &command.NotFoundError{}) - require.ErrorContains(t, err, "container not found") - }) +func assertImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + inspect, err := dockerClient.Inspect(t.Context(), imageRef) + assert.NoError(t, err, "Failed to inspect image %q", imageRef) + assert.NotNil(t, inspect, "Image should exist") +} + +func assertNoImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + inspect, err := dockerClient.Inspect(t.Context(), imageRef) + assert.ErrorIs(t, err, &command.NotFoundError{}, "Image should not exist") + assert.Nil(t, inspect, "Image should not exist") } diff --git a/pkg/registry_testhelpers/registry_container.go b/pkg/registry_testhelpers/registry_container.go index c64e0e2094..1766740957 100644 --- a/pkg/registry_testhelpers/registry_container.go +++ b/pkg/registry_testhelpers/registry_container.go @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" + "github.com/google/go-containerregistry/pkg/crane" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/registry" @@ -75,3 +76,16 @@ func (c *RegistryContainer) ImageRefForTest(t *testing.T, label string) string { repo := strings.ToLower(t.Name()) return c.ImageRef(fmt.Sprintf("%s:%s", repo, label)) } + +func (c *RegistryContainer) CloneRepo(t *testing.T, existingRepo, newRepo string) string { + existingRepo = c.ImageRef(existingRepo) + newRepo = c.ImageRef(newRepo) + + err := crane.CopyRepository(existingRepo, newRepo) + require.NoError(t, err, "Failed to clone repo %q to %q", existingRepo, newRepo) + return newRepo +} + +func (c *RegistryContainer) CloneRepoForTest(t *testing.T, repo string) string { + return c.CloneRepo(t, repo, strings.ToLower(t.Name())) +} From 78200441fb44788756c27f29645df4514430eea2 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Tue, 13 May 2025 18:05:34 -0600 Subject: [PATCH 02/43] note to self --- pkg/docker/api_client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 7b4346e988..06f33dddb2 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -22,6 +22,9 @@ import ( func NewAPIClient(ctx context.Context) (*apiClient, error) { // clicfg := cliconfig.FromContext(ctx) + // TODO[md]: we create a client at the top of each cli invocation, the sdk client hits an api which + // adds (a tiny biy of) overead. swap this with a handle that'll lazily initialize a client and ping for health. + // ditto for fetching registry credentials. client, err := dc.NewClientWithOpts(dc.FromEnv, dc.WithAPIVersionNegotiation()) if err != nil { return nil, fmt.Errorf("error creating docker client: %w", err) From 6dbfdb10460cde7f74e3f44c94ec08cedf27b68f Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Tue, 13 May 2025 22:06:10 -0600 Subject: [PATCH 03/43] fix container log multiplexing The container log output format depends on wether tty is enabled or not. - when enabled logs are a single stream from stdout - when not enabled stdout and stderr streams are multiplexed into a single stream with 8 byte headers on each line --- pkg/docker/api_client.go | 37 ++++++++++------------ pkg/docker/docker_client_test.go | 43 +++++++++++++++++++++++--- pkg/docker/dockertest/helper_client.go | 5 ++- 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 06f33dddb2..64cd013b42 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -1,7 +1,6 @@ package docker import ( - "bufio" "context" "fmt" "io" @@ -12,6 +11,7 @@ import ( "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" dc "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" "github.com/replicate/go/types/ptr" @@ -119,6 +119,12 @@ func (c *apiClient) ContainerInspect(ctx context.Context, containerID string) (* func (c *apiClient) ContainerLogs(ctx context.Context, containerID string, w io.Writer) error { console.Debugf("=== APIClient.ContainerLogs %s", containerID) + // First inspect the container to check if it has TTY enabled + inspect, err := c.ContainerInspect(ctx, containerID) + if err != nil { + return err + } + logs, err := c.client.ContainerLogs(ctx, containerID, container.LogsOptions{ ShowStdout: true, ShowStderr: true, @@ -131,27 +137,16 @@ func (c *apiClient) ContainerLogs(ctx context.Context, containerID string, w io. return fmt.Errorf("failed to get container logs for %q: %w", containerID, err) } defer logs.Close() - // Docker adds a header to each log line. The header is 8 bytes: - // - First byte is the stream type (1 for stdout, 2 for stderr) - // - Next 3 bytes are reserved - // - Last 4 bytes are the size of the message - // We want to strip the header and prefix each line with the stream type. Maybe... - // the CLI doesn't do any of this, so we might not need to do anything fancy. /shrug - scanner := bufio.NewScanner(logs) - for scanner.Scan() { - line := scanner.Text() - if len(line) < 8 { - continue - } - stream := line[0] - switch stream { - case '\x01': - fmt.Fprintln(w, "[stdout]", line[8:]) - case '\x02': - fmt.Fprintln(w, "[stderr]", line[8:]) - } + + // If TTY is enabled, we can just copy the logs directly + if inspect.Config.Tty { + _, err = io.Copy(w, logs) + return err } - return nil + + // For non-TTY containers, use StdCopy to demultiplex stdout and stderr + _, err = stdcopy.StdCopy(w, w, logs) + return err } func (c *apiClient) Push(ctx context.Context, ref string) error { diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index 85530da572..499c5e015b 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/docker/docker/api/types/container" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -220,7 +221,10 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { t.Context(), testRegistry.ImageRef("alpine:latest"), // print "line $i" N times then exit, where $i is the line number - testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"line $i\"; sleep 1; done"), + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"$i\"; sleep 1; done"), + // testcontainers.WithConfigModifier(func(config *container.Config) { + // config.Tty = true + // }), ) require.NoError(t, err, "Failed to run container") defer testcontainers.CleanupContainer(t, container) @@ -229,7 +233,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) require.NoError(t, err, "Failed to get container logs") - assert.Equal(t, "[stdout] line 1\n[stdout] line 2\n[stdout] line 3\n[stdout] line 4\n[stdout] line 5\n", buf.String()) + assert.Equal(t, "1\n2\n3\n4\n5\n", buf.String()) }) t.Run("ContainerAlreadyStopped", func(t *testing.T) { @@ -238,7 +242,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { container, err := testcontainers.Run( t.Context(), testRegistry.ImageRef("alpine:latest"), - testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 3); do echo \"line $i\"; sleep 0.1; done"), + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 3); do echo \"$i\"; sleep 0.1; done"), testcontainers.WithWaitStrategy(wait.ForExit()), ) require.NoError(t, err, "Failed to run container") @@ -252,7 +256,38 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) require.NoError(t, err, "Failed to get container logs") - assert.Equal(t, "[stdout] line 1\n[stdout] line 2\n[stdout] line 3\n", buf.String()) + assert.Equal(t, "1\n2\n3\n", buf.String()) + }) + + t.Run("TTY and non-TTY streams match", func(t *testing.T) { + t.Parallel() + + runContainer := func(tty bool) string { + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + // print "line $i" N times then exit, where $i is the line number + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"$i\"; sleep 0.1; done"), + testcontainers.WithConfigModifier(func(config *container.Config) { + config.Tty = tty + }), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + return buf.String() + } + + ttyOutput := runContainer(true) + nonTtyOutput := runContainer(false) + + // TTY uses CRLF for line endings, non-TTY uses LF. replace \r\n with \n so they match + ttyOutput = strings.ReplaceAll(ttyOutput, "\r\n", "\n") + + assert.Equal(t, ttyOutput, nonTtyOutput, "TTY and non-TTY streams should match after normalizing line endings") }) t.Run("ContainerDoesNotExist", func(t *testing.T) { diff --git a/pkg/docker/dockertest/helper_client.go b/pkg/docker/dockertest/helper_client.go index 5e24c9e756..d67eb2bc04 100644 --- a/pkg/docker/dockertest/helper_client.go +++ b/pkg/docker/dockertest/helper_client.go @@ -210,10 +210,13 @@ func (c *HelperClient) CleanupImage(t testing.TB, imageRef string) { t.Helper() t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), imageRef, image.RemoveOptions{ + _, err := c.Client.ImageRemove(context.Background(), imageRef, image.RemoveOptions{ Force: true, PruneChildren: true, }) + if err != nil { + t.Logf("Warning: Failed to remove image %q: %v", imageRef, err) + } }) } From dd16c9aa4872d01f5fc3b54f3b6e8570bb544769 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 14 May 2025 08:47:20 -0600 Subject: [PATCH 04/43] return correct not found error from logs --- pkg/docker/docker_command.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 6b1c4d6663..f49f149bb9 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -207,7 +207,14 @@ func (c *DockerCommand) ContainerLogs(ctx context.Context, containerID string, w "--follow", } - return c.exec(ctx, nil, w, nil, "", args) + err := c.exec(ctx, nil, w, nil, "", args) + if err != nil { + if strings.Contains(err.Error(), "No such container") { + return &command.NotFoundError{Ref: containerID, Object: "container"} + } + return err + } + return err } func (c *DockerCommand) ContainerInspect(ctx context.Context, id string) (*container.InspectResponse, error) { From 1fb618b5e9b06b6fd63ed98af15cebec69e3e57b Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 14 May 2025 17:06:46 -0600 Subject: [PATCH 05/43] test registry helpers for auth, image inspection --- .../registry_container.go | 85 +++++++++++++++++-- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/pkg/registry_testhelpers/registry_container.go b/pkg/registry_testhelpers/registry_container.go index 1766740957..3e6ec35270 100644 --- a/pkg/registry_testhelpers/registry_container.go +++ b/pkg/registry_testhelpers/registry_container.go @@ -12,11 +12,17 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/registry" "github.com/testcontainers/testcontainers-go/wait" + "golang.org/x/crypto/bcrypt" + + dockerregistry "github.com/docker/docker/api/types/registry" "github.com/replicate/cog/pkg/util" ) @@ -26,23 +32,26 @@ import ( // that can be used to inspect the registry and generate absolute image references. It will // automatically be cleaned when the test finishes. // This is safe to run concurrently across multiple tests. -func StartTestRegistry(t *testing.T) *RegistryContainer { +func StartTestRegistry(t *testing.T, opts ...Option) *RegistryContainer { t.Helper() + options := &options{} + for _, opt := range opts { + opt(options) + } + _, filename, _, _ := runtime.Caller(0) testdataDir := filepath.Join(filepath.Dir(filename), "testdata", "docker") - registryContainer, err := registry.Run( - t.Context(), - "registry:3", + containerCustomizers := []testcontainers.ContainerCustomizer{ testcontainers.WithFiles(testcontainers.ContainerFile{ HostFilePath: testdataDir, ContainerFilePath: "/var/lib/registry/", FileMode: 0o755, }), testcontainers.WithWaitStrategy( - wait.ForHTTP("/v2/").WithPort("5000/tcp"). - WithStartupTimeout(10*time.Second), + wait.ForHTTP("/").WithPort("5000/tcp"). + WithStartupTimeout(10 * time.Second), ), testcontainers.WithHostConfigModifier(func(hostConfig *container.HostConfig) { // docker only considers localhost:1 through localhost:9999 as insecure. testcontainers @@ -54,15 +63,33 @@ func StartTestRegistry(t *testing.T) *RegistryContainer { nat.Port("5000/tcp"): {{HostIP: "0.0.0.0", HostPort: strconv.Itoa(port)}}, } }), + } + + if options.auth != nil { + htpasswd, err := generateHtpasswd(options.auth.Username, options.auth.Password) + require.NoError(t, err) + containerCustomizers = append(containerCustomizers, + registry.WithHtpasswd(htpasswd), + ) + } + + registryContainer, err := registry.Run( + t.Context(), + "registry:3", + containerCustomizers..., ) defer testcontainers.CleanupContainer(t, registryContainer) require.NoError(t, err, "Failed to start registry container") - return &RegistryContainer{Container: registryContainer} + return &RegistryContainer{ + Container: registryContainer, + options: options, + } } type RegistryContainer struct { Container *registry.RegistryContainer + options *options } func (c *RegistryContainer) ImageRef(ref string) string { @@ -89,3 +116,47 @@ func (c *RegistryContainer) CloneRepo(t *testing.T, existingRepo, newRepo string func (c *RegistryContainer) CloneRepoForTest(t *testing.T, repo string) string { return c.CloneRepo(t, repo, strings.ToLower(t.Name())) } + +func (c *RegistryContainer) ImageExists(t *testing.T, ref string) error { + parsedRef, err := name.ParseReference(ref, name.WithDefaultRegistry(c.RegistryHost())) + require.NoError(t, err) + + var opts []remote.Option + + if c.options.auth != nil { + opts = append(opts, remote.WithAuth(authn.FromConfig(authn.AuthConfig{ + Username: c.options.auth.Username, + Password: c.options.auth.Password, + }))) + } + _, err = remote.Head(parsedRef, opts...) + return err +} + +func (c *RegistryContainer) RegistryHost() string { + return c.Container.RegistryName +} + +type Option func(*options) + +func WithAuth(username, password string) func(*options) { + return func(o *options) { + o.auth = &dockerregistry.AuthConfig{ + Username: username, + Password: password, + } + } +} + +type options struct { + auth *dockerregistry.AuthConfig +} + +func generateHtpasswd(username, password string) (string, error) { + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", username, string(hash)), nil + // return fmt.Sprintf("%s:$2y$05$%s", username, base64.StdEncoding.EncodeToString([]byte(password))) +} From 17d80d31953d91b9351d4e4a86abfb2a931e83d9 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 14 May 2025 17:07:46 -0600 Subject: [PATCH 06/43] test helpers for image refs, concurrent loads --- pkg/docker/dockertest/helper_client.go | 74 +++++++++++++++++------ pkg/docker/dockertest/ref.go | 82 ++++++++++++++++++++++++++ pkg/docker/dockertest/ref_test.go | 24 ++++++++ 3 files changed, 163 insertions(+), 17 deletions(-) create mode 100644 pkg/docker/dockertest/ref.go create mode 100644 pkg/docker/dockertest/ref_test.go diff --git a/pkg/docker/dockertest/helper_client.go b/pkg/docker/dockertest/helper_client.go index d67eb2bc04..cd6f7c49ab 100644 --- a/pkg/docker/dockertest/helper_client.go +++ b/pkg/docker/dockertest/helper_client.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "slices" + "sync" "testing" "github.com/docker/docker/api/types/container" @@ -41,17 +42,33 @@ func NewHelperClient(t testing.TB) *HelperClient { t.Skip("Docker daemon is not running") } + helper := &HelperClient{ + Client: cli, + fixtures: make(map[string]*imageFixture), + mu: &sync.Mutex{}, + } + t.Cleanup(func() { + for _, img := range helper.fixtures { + _, err := helper.Client.ImageRemove(context.Background(), img.imageID, image.RemoveOptions{Force: true, PruneChildren: true}) + if err != nil { + t.Logf("Warning: Failed to remove image %q: %v", img.imageID, err) + } + } + if err := cli.Close(); err != nil { t.Fatalf("Failed to close Docker client: %v", err) } }) - return &HelperClient{Client: cli} + return helper } type HelperClient struct { Client *client.Client + + mu *sync.Mutex + fixtures map[string]*imageFixture } func (c *HelperClient) Close() error { @@ -258,8 +275,31 @@ func (c *HelperClient) InspectContainer(t testing.TB, containerID string) *conta return &inspect } -func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { +func (c *HelperClient) ImageFixture(t testing.TB, name string, tag string) { t.Helper() + fixture := c.loadImageFixture(t, name) + + t.Logf("Tagging image fixture %q with %q", fixture.ref, tag) + if err := c.Client.ImageTag(t.Context(), fixture.imageID, tag); err != nil { + require.NoError(t, err, "Failed to tag image %q with %q: %v", fixture.ref, tag, err) + } + // remove the image when the test is done + t.Cleanup(func() { + _, _ = c.Client.ImageRemove(context.Background(), tag, image.RemoveOptions{Force: true}) + }) +} + +func (c *HelperClient) loadImageFixture(t testing.TB, name string) *imageFixture { + t.Helper() + + c.mu.Lock() + defer c.mu.Unlock() + + ref := fmt.Sprintf("cog-test-fixture:%s", name) + + if fixture, ok := c.fixtures[ref]; ok { + return fixture + } // Get the path of the current file _, filename, _, ok := runtime.Caller(0) @@ -273,7 +313,6 @@ func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { // Construct the path to the fixture fixturePath := filepath.Join(dir, "testdata", name+".tar") - ref := fmt.Sprintf("cog-test-fixture:%s", name) t.Logf("Loading image fixture %q from %s", ref, fixturePath) f, err := os.Open(fixturePath) @@ -283,22 +322,23 @@ func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { l, err := c.Client.ImageLoad(t.Context(), f) require.NoError(t, err, "Failed to load fixture %q", name) defer l.Body.Close() - _, err = io.Copy(os.Stdout, l.Body) + _, err = io.Copy(os.Stderr, l.Body) require.NoError(t, err, "Failed to copy fixture %q", name) - // remove the image when the test is done - t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), ref, image.RemoveOptions{}) - }) + inspect, err := c.Client.ImageInspect(t.Context(), ref) + require.NoError(t, err, "Failed to inspect image %q", ref) - if tag != "" { - t.Logf("Tagging image fixture %q with %q", ref, tag) - if err := c.Client.ImageTag(t.Context(), ref, tag); err != nil { - require.NoError(t, err, "Failed to tag image %q with %q: %v", ref, tag, err) - } - // remove the image when the test is done - t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), tag, image.RemoveOptions{}) - }) + fixture := &imageFixture{ + ref: ref, + imageID: inspect.ID, } + + c.fixtures[ref] = fixture + + return fixture +} + +type imageFixture struct { + imageID string + ref string } diff --git a/pkg/docker/dockertest/ref.go b/pkg/docker/dockertest/ref.go new file mode 100644 index 0000000000..4b7d0fe9e3 --- /dev/null +++ b/pkg/docker/dockertest/ref.go @@ -0,0 +1,82 @@ +package dockertest + +import ( + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/stretchr/testify/require" +) + +type Ref struct { + t *testing.T + ref name.Reference +} + +func NewRef(t *testing.T) Ref { + t.Helper() + + repoName := strings.ToLower(t.Name()) + // Replace any characters that aren't valid in a docker image repo name with underscore + // Valid characters are: a-z, 0-9, ., _, -, / + repoName = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' || r == '/' { + return r + } + return '_' + }, repoName) + + ref, err := name.ParseReference(repoName, name.WithDefaultRegistry("")) + require.NoError(t, err, "Failed to create reference for test") + + return Ref{t: t, ref: ref} +} + +func (r Ref) WithTag(tagName string) Ref { + tagRef := r.ref.Context().Tag(tagName) + return Ref{t: r.t, ref: tagRef} +} + +func (r Ref) WithDigest(digest string) Ref { + digestRef := r.ref.Context().Digest(digest) + return Ref{t: r.t, ref: digestRef} +} + +func (r Ref) WithRegistry(registry string) Ref { + reg, err := name.NewRegistry(registry) + require.NoError(r.t, err, "Failed to create registry for test") + + repo := r.ref.Context() + repo.Registry = reg + var newRef name.Reference + switch r.ref.(type) { + case name.Tag: + newRef = repo.Tag(r.ref.Identifier()) + case name.Digest: + newRef = repo.Digest(r.ref.Identifier()) + default: + require.Fail(r.t, "Unsupported reference type") + } + + return Ref{t: r.t, ref: newRef} +} + +func (r Ref) WithoutRegistry() Ref { + repo := r.ref.Context() + repo.Registry = name.Registry{} + var newRef name.Reference + switch r.ref.(type) { + case name.Tag: + newRef = repo.Tag(r.ref.Identifier()) + case name.Digest: + newRef = repo.Digest(r.ref.Identifier()) + default: + require.Fail(r.t, "Unsupported reference type") + } + + return Ref{t: r.t, ref: newRef} +} + +func (r Ref) String() string { + return r.ref.Name() +} diff --git a/pkg/docker/dockertest/ref_test.go b/pkg/docker/dockertest/ref_test.go new file mode 100644 index 0000000000..754b12dec8 --- /dev/null +++ b/pkg/docker/dockertest/ref_test.go @@ -0,0 +1,24 @@ +package dockertest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRef(t *testing.T) { + ref := NewRef(t) + assert.Equal(t, "testref:latest", ref.String()) + + ref = ref.WithTag("v2") + assert.Equal(t, "testref:v2", ref.String()) + + ref = ref.WithRegistry("r8.im") + assert.Equal(t, "r8.im/testref:v2", ref.String()) + + ref = ref.WithoutRegistry() + assert.Equal(t, "testref:v2", ref.String()) + + ref = ref.WithDigest("sha256:71859b0c62df47efaeae4f93698b56a8dddafbf041778fd668bbd1ab45a864f8") + assert.Equal(t, "testref@sha256:71859b0c62df47efaeae4f93698b56a8dddafbf041778fd668bbd1ab45a864f8", ref.String()) +} From 15f1f00d02487e76197a8db8cd88ff1a426696e4 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 14 May 2025 17:08:05 -0600 Subject: [PATCH 07/43] api client push, authentication support --- pkg/docker/api_client.go | 95 ++++++++++++++++++++--- pkg/docker/command/errors.go | 2 + pkg/docker/docker_client_test.go | 126 ++++++++++++++++++++++++++++--- pkg/docker/docker_command.go | 16 +++- 4 files changed, 217 insertions(+), 22 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 64cd013b42..1422001783 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -2,16 +2,20 @@ package docker import ( "context" + "errors" "fmt" "io" "os" + "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" dc "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" + "github.com/google/go-containerregistry/pkg/name" "github.com/replicate/go/types/ptr" @@ -19,8 +23,13 @@ import ( "github.com/replicate/cog/pkg/util/console" ) -func NewAPIClient(ctx context.Context) (*apiClient, error) { - // clicfg := cliconfig.FromContext(ctx) +func NewAPIClient(ctx context.Context, opts ...Option) (*apiClient, error) { + clientOptions := &clientOptions{ + authConfigs: make(map[string]registry.AuthConfig), + } + for _, opt := range opts { + opt(clientOptions) + } // TODO[md]: we create a client at the top of each cli invocation, the sdk client hits an api which // adds (a tiny biy of) overead. swap this with a handle that'll lazily initialize a client and ping for health. @@ -40,8 +49,13 @@ func NewAPIClient(ctx context.Context) (*apiClient, error) { return nil, fmt.Errorf("error loading user information: %w", err) } authConfig["r8.im"] = registry.AuthConfig{ - Username: userInfo.Username, - Password: userInfo.Token, + Username: userInfo.Username, + Password: userInfo.Token, + ServerAddress: "r8.im", + } + + for _, opt := range clientOptions.authConfigs { + authConfig[opt.ServerAddress] = opt } return &apiClient{client, authConfig}, nil @@ -78,7 +92,10 @@ func (c *apiClient) Pull(ctx context.Context, imageRef string, force bool) (*ima return nil, fmt.Errorf("failed to pull image %q: %w", imageRef, err) } defer output.Close() - io.Copy(os.Stderr, output) + _, err = io.Copy(os.Stderr, output) + if err != nil { + return nil, fmt.Errorf("failed to copy pull output: %w", err) + } // pull succeeded, inspect the image again and return inspect, err := c.Inspect(ctx, imageRef) @@ -122,7 +139,7 @@ func (c *apiClient) ContainerLogs(ctx context.Context, containerID string, w io. // First inspect the container to check if it has TTY enabled inspect, err := c.ContainerInspect(ctx, containerID) if err != nil { - return err + return fmt.Errorf("failed to inspect container %q: %w", containerID, err) } logs, err := c.client.ContainerLogs(ctx, containerID, container.LogsOptions{ @@ -140,17 +157,69 @@ func (c *apiClient) ContainerLogs(ctx context.Context, containerID string, w io. // If TTY is enabled, we can just copy the logs directly if inspect.Config.Tty { - _, err = io.Copy(w, logs) - return err + if _, err = io.Copy(w, logs); err != nil { + return fmt.Errorf("failed to copy logs: %w", err) + } + return nil } // For non-TTY containers, use StdCopy to demultiplex stdout and stderr - _, err = stdcopy.StdCopy(w, w, logs) - return err + if _, err = stdcopy.StdCopy(w, w, logs); err != nil { + return fmt.Errorf("failed to copy logs: %w", err) + } + return nil } -func (c *apiClient) Push(ctx context.Context, ref string) error { - panic("not implemented") +func (c *apiClient) Push(ctx context.Context, imageRef string) error { + console.Debugf("=== APIClient.Push %s", imageRef) + + parsedName, err := name.ParseReference(imageRef) + if err != nil { + return fmt.Errorf("failed to parse image reference: %w", err) + } + + console.Debugf("fully qualified image ref: %s", parsedName) + + // eagerly set auth config, or do it async + var authConfig registry.AuthConfig + if auth, ok := c.authConfig[parsedName.Context().RegistryStr()]; ok { + authConfig = auth + } else { + console.Warnf("no auth config found for registry %s", parsedName.Context().RegistryStr()) + } + + var opts image.PushOptions + encodedAuth, err := registry.EncodeAuthConfig(authConfig) + if err != nil { + return fmt.Errorf("failed to encode auth config: %w", err) + } + opts.RegistryAuth = encodedAuth + + output, err := c.client.ImagePush(ctx, imageRef, opts) + if err != nil { + return fmt.Errorf("failed to push image: %w", err) + } + defer output.Close() + + // output is a json stream, so we need to parse it, handle errors, and write progress to stderr + isTTY := console.IsTTY(os.Stderr) + if err := jsonmessage.DisplayJSONMessagesStream(output, os.Stderr, os.Stderr.Fd(), isTTY, nil); err != nil { + var streamErr *jsonmessage.JSONError + if errors.As(err, &streamErr) { + if strings.Contains(streamErr.Message, "tag does not exist") { + return &command.NotFoundError{Ref: imageRef, Object: "tag"} + } + if strings.Contains(streamErr.Message, "authorization failed") { + return command.ErrAuthorizationFailed + } + if strings.Contains(streamErr.Message, "401 Unauthorized") { + return command.ErrAuthorizationFailed + } + } + return fmt.Errorf("error during image push: %w", err) + } + + return nil } func (c *apiClient) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { @@ -167,6 +236,8 @@ func (c *apiClient) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarF } func (c *apiClient) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) { + console.Debugf("=== APIClient.Inspect %s", ref) + // TODO[md]: platform requires engine 1.49+, and it's not widly available as of 2025-05. // platform := ocispec.Platform{OS: "linux", Architecture: "amd64"} // client.ImageInspectWithPlatform(&platform), diff --git a/pkg/docker/command/errors.go b/pkg/docker/command/errors.go index 8bcc4ba9eb..66f35a1098 100644 --- a/pkg/docker/command/errors.go +++ b/pkg/docker/command/errors.go @@ -29,3 +29,5 @@ func (e *NotFoundError) Is(target error) bool { func IsNotFoundError(err error) bool { return errors.Is(err, &NotFoundError{}) } + +var ErrAuthorizationFailed = errors.New("authorization failed") diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index 499c5e015b..f526cfa010 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -2,12 +2,13 @@ package docker import ( "bytes" - "fmt" + "context" "strings" "testing" "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -50,12 +51,12 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { t.Run("ExistingLocalImage", func(t *testing.T) { t.Parallel() - imageRef := imageRefForTest(t) - dockerHelper.LoadImageFixture(t, "alpine", imageRef) + ref := dockertest.NewRef(t) + dockerHelper.ImageFixture(t, "alpine", ref.String()) - expectedImage := dockerHelper.InspectImage(t, imageRef) - resp, err := dockerClient.Inspect(t.Context(), imageRef) - require.NoError(t, err, "Failed to inspect image %q", imageRef) + expectedImage := dockerHelper.InspectImage(t, ref.String()) + resp, err := dockerClient.Inspect(t.Context(), ref.String()) + require.NoError(t, err, "Failed to inspect image %q", ref.String()) assert.Equal(t, expectedImage.ID, resp.ID) }) @@ -297,20 +298,127 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { require.ErrorIs(t, err, &command.NotFoundError{}) }) }) -} -func imageRefForTest(t *testing.T) string { - return fmt.Sprintf("%s:test-%d", strings.ToLower(t.Name()), time.Now().Unix()) + t.Run("Push", func(t *testing.T) { + t.Parallel() + + t.Run("valid image, valid registry", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(testRegistry.RegistryHost()) + + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + err := dockerClient.Push(t.Context(), ref.String()) + require.NoError(t, err) + assert.NoError(t, testRegistry.ImageExists(t, ref.String())) + }) + + t.Run("non-existent registry", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry("localhost:1234") + dockerHelper.ImageFixture(t, "alpine", ref.String()) + // should timeout trying to connect to a bogus registry + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + err := dockerClient.Push(ctx, ref.String()) + require.ErrorIs(t, err, context.DeadlineExceeded, "should timeout trying to connect to a bogus registry") + }) + + t.Run("missing image", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(testRegistry.RegistryHost()) + + err := dockerClient.Push(t.Context(), ref.String()) + assertNotFoundError(t, err, ref.String(), "tag") + }) + + t.Run("registry with authentication", func(t *testing.T) { + t.Parallel() + + if _, ok := dockerClient.(*DockerCommand); ok { + t.Skip("skipping auth tests for docker command client since we can't set auth on the host without side effects") + } + + authReg := registry_testhelpers.StartTestRegistry(t, registry_testhelpers.WithAuth("testuser", "testpass")) + + t.Run("correct credentials", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + // create a new client with the correct auth config + authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + Username: "testuser", + Password: "testpass", + ServerAddress: authReg.RegistryHost(), + })) + require.NoError(t, err) + + err = authClient.Push(t.Context(), ref.String()) + require.NoError(t, err, "Failed to push image to auth registry") + assert.NoError(t, authReg.ImageExists(t, ref.String())) + }) + + t.Run("missing auth", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + // use root client which doesn't have auth setup + err := dockerClient.Push(t.Context(), ref.String()) + require.ErrorIs(t, err, command.ErrAuthorizationFailed) + }) + + t.Run("incorrect auth", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + Username: "testuser", + Password: "wrongpass", + ServerAddress: authReg.RegistryHost(), + })) + require.NoError(t, err) + + err = authClient.Push(t.Context(), ref.String()) + require.ErrorIs(t, err, command.ErrAuthorizationFailed) + }) + + t.Run("correct credentials, not authorized", func(t *testing.T) { + t.Skip("skipping until the registry supports repo authorizations") + }) + }) + }) } func assertImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + t.Helper() + inspect, err := dockerClient.Inspect(t.Context(), imageRef) assert.NoError(t, err, "Failed to inspect image %q", imageRef) assert.NotNil(t, inspect, "Image should exist") } func assertNoImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + t.Helper() + inspect, err := dockerClient.Inspect(t.Context(), imageRef) assert.ErrorIs(t, err, &command.NotFoundError{}, "Image should not exist") assert.Nil(t, inspect, "Image should not exist") } + +func assertNotFoundError(t *testing.T, err error, ref string, object string) { + t.Helper() + + var notFoundErr *command.NotFoundError + require.ErrorAs(t, err, ¬FoundErr, "should be a not found error") + require.Equal(t, ref, notFoundErr.Ref, "ref should match") + require.Equal(t, object, notFoundErr.Object, "object should match") +} diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index f49f149bb9..79fd2fd781 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -77,7 +77,15 @@ func (c *DockerCommand) Pull(ctx context.Context, image string, force bool) (*im func (c *DockerCommand) Push(ctx context.Context, image string) error { console.Debugf("=== DockerCommand.Push %s", image) - return c.exec(ctx, nil, nil, nil, "", []string{"push", image}) + err := c.exec(ctx, nil, nil, nil, "", []string{"push", image}) + if err != nil { + if strings.Contains(err.Error(), "tag does not exist") { + return &command.NotFoundError{Ref: image, Object: "tag"} + } + return err + } + + return nil } func (c *DockerCommand) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { @@ -497,6 +505,12 @@ func (c *DockerCommand) exec(ctx context.Context, in io.Reader, outw, errw io.Wr if errors.Is(err, context.Canceled) { return err } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + if !exitErr.Exited() && strings.Contains(exitErr.Error(), "signal: killed") { + return context.DeadlineExceeded + } + } return fmt.Errorf("command failed: %s: %w", stderrBuf.String(), err) } return nil From 88131501614432a7f933cc140e8a2bf1f69266bf Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 14 May 2025 17:41:55 -0600 Subject: [PATCH 08/43] helpers to map backend errors --- pkg/docker/api_client.go | 10 ++------ pkg/docker/docker_command.go | 15 ++++++----- pkg/docker/errors.go | 49 ++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 pkg/docker/errors.go diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 1422001783..97ee2da4e6 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" @@ -184,8 +183,6 @@ func (c *apiClient) Push(ctx context.Context, imageRef string) error { var authConfig registry.AuthConfig if auth, ok := c.authConfig[parsedName.Context().RegistryStr()]; ok { authConfig = auth - } else { - console.Warnf("no auth config found for registry %s", parsedName.Context().RegistryStr()) } var opts image.PushOptions @@ -206,13 +203,10 @@ func (c *apiClient) Push(ctx context.Context, imageRef string) error { if err := jsonmessage.DisplayJSONMessagesStream(output, os.Stderr, os.Stderr.Fd(), isTTY, nil); err != nil { var streamErr *jsonmessage.JSONError if errors.As(err, &streamErr) { - if strings.Contains(streamErr.Message, "tag does not exist") { + if isTagNotFoundError(err) { return &command.NotFoundError{Ref: imageRef, Object: "tag"} } - if strings.Contains(streamErr.Message, "authorization failed") { - return command.ErrAuthorizationFailed - } - if strings.Contains(streamErr.Message, "401 Unauthorized") { + if isAuthorizationFailedError(err) { return command.ErrAuthorizationFailed } } diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 79fd2fd781..96c29e88c6 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -79,9 +79,12 @@ func (c *DockerCommand) Push(ctx context.Context, image string) error { err := c.exec(ctx, nil, nil, nil, "", []string{"push", image}) if err != nil { - if strings.Contains(err.Error(), "tag does not exist") { + if isTagNotFoundError(err) { return &command.NotFoundError{Ref: image, Object: "tag"} } + if isAuthorizationFailedError(err) { + return command.ErrAuthorizationFailed + } return err } @@ -170,7 +173,7 @@ func (c *DockerCommand) Inspect(ctx context.Context, ref string) (*image.Inspect } output, err := c.execCaptured(ctx, nil, "", args) if err != nil { - if strings.Contains(err.Error(), "No such image") { + if isImageNotFoundError(err) { return nil, &command.NotFoundError{Object: "image", Ref: ref} } return nil, err @@ -217,7 +220,7 @@ func (c *DockerCommand) ContainerLogs(ctx context.Context, containerID string, w err := c.exec(ctx, nil, w, nil, "", args) if err != nil { - if strings.Contains(err.Error(), "No such container") { + if isContainerNotFoundError(err) { return &command.NotFoundError{Ref: containerID, Object: "container"} } return err @@ -236,7 +239,7 @@ func (c *DockerCommand) ContainerInspect(ctx context.Context, id string) (*conta output, err := c.execCaptured(ctx, nil, "", args) if err != nil { - if strings.Contains(err.Error(), "No such container") { + if isContainerNotFoundError(err) { return nil, &command.NotFoundError{Object: "container", Ref: id} } return nil, err @@ -264,7 +267,7 @@ func (c *DockerCommand) ContainerStop(ctx context.Context, containerID string) e } if err := c.exec(ctx, nil, nil, nil, "", args); err != nil { - if strings.Contains(err.Error(), "No such container") { + if isContainerNotFoundError(err) { err = &command.NotFoundError{Object: "container", Ref: containerID} } return fmt.Errorf("failed to stop container %q: %w", containerID, err) @@ -432,7 +435,7 @@ func (c *DockerCommand) containerRun(ctx context.Context, options command.RunOpt err := c.exec(ctx, options.Stdin, options.Stdout, options.Stderr, "", args) if err != nil { - if strings.Contains(err.Error(), "could not select device driver") || strings.Contains(err.Error(), "nvidia-container-cli: initialization error") { + if isMissingDeviceDriverError(err) { return ErrMissingDeviceDriver } return err diff --git a/pkg/docker/errors.go b/pkg/docker/errors.go new file mode 100644 index 0000000000..feac88a465 --- /dev/null +++ b/pkg/docker/errors.go @@ -0,0 +1,49 @@ +package docker + +import "strings" + +// Error messages vary between different backends (dockerd, containerd, podman, orbstack, etc) or even versions of docker. +// These helpers normalize the check so callers can handle situations without worrying about the underlying implementation. +// Yes, it's gross, but whattaya gonna do + +func isTagNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "tag does not exist") || + strings.Contains(msg, "An image does not exist locally with the tag") +} + +func isImageNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "image does not exist") || + strings.Contains(msg, "No such image") +} + +func isContainerNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "container does not exist") || + strings.Contains(msg, "No such container") +} + +func isAuthorizationFailedError(err error) bool { + msg := err.Error() + + // registry requires auth and none were provided + if strings.Contains(msg, "no basic auth credentials") { + return true + } + + // registry rejected the provided auth + if strings.Contains(msg, "authorization failed") || + strings.Contains(msg, "401 Unauthorized") || + strings.Contains(msg, "unauthorized: authentication required") { + return true + } + + return false +} + +func isMissingDeviceDriverError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "could not select device driver") || + strings.Contains(msg, "nvidia-container-cli: initialization error") +} From 238c7022aa3216f991bee2e0559fd50704c5629c Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 10:17:35 -0600 Subject: [PATCH 09/43] fix test for connection refused registry --- pkg/docker/docker_client_test.go | 36 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index f526cfa010..d1a3c617b1 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -2,10 +2,10 @@ package docker import ( "bytes" - "context" + "net" + "strconv" "strings" "testing" - "time" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/registry" @@ -18,6 +18,7 @@ import ( "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/docker/dockertest" "github.com/replicate/cog/pkg/registry_testhelpers" + "github.com/replicate/cog/pkg/util" ) func TestDockerClient(t *testing.T) { @@ -317,13 +318,32 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { t.Run("non-existent registry", func(t *testing.T) { t.Parallel() - ref := dockertest.NewRef(t).WithRegistry("localhost:1234") + // start a local tcp server that immediately closes connections + port, err := util.PickFreePort(2000, 9999) + require.NoError(t, err, "Failed to pick free tcp port") + addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) + listener, err := net.Listen("tcp", addr) + require.NoError(t, err) + defer listener.Close() + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + conn.Close() + } + }() + + // Create a reference to the mock registry + ref := dockertest.NewRef(t).WithRegistry(addr) dockerHelper.ImageFixture(t, "alpine", ref.String()) - // should timeout trying to connect to a bogus registry - ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) - defer cancel() - err := dockerClient.Push(ctx, ref.String()) - require.ErrorIs(t, err, context.DeadlineExceeded, "should timeout trying to connect to a bogus registry") + + // Try to push to the mock registry + err = dockerClient.Push(t.Context(), ref.String()) + require.Error(t, err, "Push should fail with unreachable registry") + assert.ErrorContains(t, err, "connection refused", "Error should indicate registry is unreachable") }) t.Run("missing image", func(t *testing.T) { From ef44dd4a4645d329e86457fa8815e2bcf2acd4ec Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 10:33:05 -0600 Subject: [PATCH 10/43] fix assertions for both dev and CI --- pkg/docker/docker_client_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index d1a3c617b1..048ecb731f 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -343,7 +343,12 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { // Try to push to the mock registry err = dockerClient.Push(t.Context(), ref.String()) require.Error(t, err, "Push should fail with unreachable registry") - assert.ErrorContains(t, err, "connection refused", "Error should indicate registry is unreachable") + + // error message varies between dev and CI host environments, cover them all... + assert.Condition(t, func() bool { + msg := err.Error() + return strings.Contains(msg, "connection refused") || strings.Contains(msg, "EOF") + }, "Error should indicate registry is unreachable") }) t.Run("missing image", func(t *testing.T) { From e75dd828d675f1d086fc56072bdb522d7b49ca0d Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 11:42:55 -0600 Subject: [PATCH 11/43] move user info logic off command, share with api --- pkg/docker/api_client.go | 3 +- pkg/docker/credentials.go | 55 ++++++++++++++++++ pkg/docker/docker_command.go | 108 +---------------------------------- pkg/docker/login.go | 2 +- 4 files changed, 60 insertions(+), 108 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 97ee2da4e6..7c26b6b43f 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -216,9 +216,9 @@ func (c *apiClient) Push(ctx context.Context, imageRef string) error { return nil } +// TODO[md]: this doesn't need to be on the interface, move to auth handler func (c *apiClient) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== APIClient.LoadUserInformation %s", registryHost) - panic("not implemented") } func (c *apiClient) CreateTarFile(ctx context.Context, ref string, tmpDir string, tarFile string, folder string) (string, error) { @@ -227,6 +227,7 @@ func (c *apiClient) CreateTarFile(ctx context.Context, ref string, tmpDir string func (c *apiClient) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) { panic("not implemented") + return loadUserInformation(ctx, registryHost) } func (c *apiClient) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) { diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go index bf4b8690c0..28b26332b9 100644 --- a/pkg/docker/credentials.go +++ b/pkg/docker/credentials.go @@ -2,11 +2,19 @@ package docker import ( "context" + "encoding/json" + "fmt" + "io" "os" + "os/exec" + "strings" "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/config/types" "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" ) func loadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { @@ -31,3 +39,50 @@ func loadUserInformation(ctx context.Context, registryHost string) (*command.Use Username: credsHelper.Username, }, nil } + +func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types.AuthConfig, error) { + return conf.AuthConfigs[registryHost], nil +} + +func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { + var out strings.Builder + binary := dockerCredentialBinary(credsStore) + cmd := exec.CommandContext(ctx, binary, "get") + cmd.Env = os.Environ() + cmd.Stdout = &out + cmd.Stderr = &out + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + defer stdin.Close() + console.Debug("$ " + strings.Join(cmd.Args, " ")) + err = cmd.Start() + if err != nil { + return nil, err + } + _, err = io.WriteString(stdin, registryHost) + if err != nil { + return nil, err + } + err = stdin.Close() + if err != nil { + return nil, err + } + err = cmd.Wait() + if err != nil { + return nil, fmt.Errorf("exec wait error: %w", err) + } + + var config CredentialHelperInput + err = json.Unmarshal([]byte(out.String()), &config) + if err != nil { + return nil, err + } + + return &config, nil +} + +func dockerCredentialBinary(credsStore string) string { + return "docker-credential-" + credsStore +} diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 96c29e88c6..29d10da52c 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -91,77 +91,20 @@ func (c *DockerCommand) Push(ctx context.Context, image string) error { return nil } +// TODO[md]: this doesn't need to be on the interface, move to auth handler func (c *DockerCommand) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== DockerCommand.LoadUserInformation %s", registryHost) conf := config.LoadDefaultConfigFile(os.Stderr) - credsStore := conf.CredentialsStore - if credsStore == "" { - authConf, err := loadAuthFromConfig(conf, registryHost) - if err != nil { - return nil, err - } - return &command.UserInfo{ - Token: authConf.Password, - Username: authConf.Username, - }, nil - } - credsHelper, err := loadAuthFromCredentialsStore(ctx, credsStore, registryHost) - if err != nil { - return nil, err - } - return &command.UserInfo{ - Token: credsHelper.Secret, - Username: credsHelper.Username, - }, nil -} - func (c *DockerCommand) CreateTarFile(ctx context.Context, image string, tmpDir string, tarFile string, folder string) (string, error) { console.Debugf("=== DockerCommand.CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) args := []string{ "run", "--rm", - // force platform to linux/amd64 so darwin/arm64 outputs work in prod - "--platform", "linux/amd64", - "--volume", - tmpDir + ":/buildtmp", - image, - "/opt/r8/monobase/tar.sh", - "/buildtmp/" + tarFile, - "/", - folder, - } - if err := c.exec(ctx, nil, nil, nil, "", args); err != nil { - return "", err - } - return filepath.Join(tmpDir, tarFile), nil -} - -func (c *DockerCommand) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) { - console.Debugf("=== DockerCommand.CreateAptTarFile %s %s", aptTarFile, packages) - - // This uses a hardcoded monobase image to produce an apt tar file. - // The reason being that this apt tar file is created outside the docker file, and it is created by - // running the apt.sh script on the monobase with the packages we intend to install, which produces - // a tar file that can be untarred into a docker build to achieve the equivalent of an apt-get install. - args := []string{ - "run", - "--rm", - // force platform to linux/amd64 so darwin/arm64 outputs work in prod - "--platform", "linux/amd64", - "--volume", tmpDir + ":/buildtmp", "r8.im/monobase:latest", - "/opt/r8/monobase/apt.sh", - "/buildtmp/" + aptTarFile, - } - args = append(args, packages...) - if err := c.exec(ctx, nil, nil, nil, "", args); err != nil { - return "", err - } - - return aptTarFile, nil + return loadUserInformation(ctx, registryHost) } func (c *DockerCommand) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) { @@ -527,50 +470,3 @@ func (c *DockerCommand) execCaptured(ctx context.Context, in io.Reader, dir stri } return out.String(), nil } - -func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types.AuthConfig, error) { - return conf.AuthConfigs[registryHost], nil -} - -func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { - var out strings.Builder - binary := DockerCredentialBinary(credsStore) - cmd := exec.CommandContext(ctx, binary, "get") - cmd.Env = os.Environ() - cmd.Stdout = &out - cmd.Stderr = &out - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - defer stdin.Close() - console.Debug("$ " + strings.Join(cmd.Args, " ")) - err = cmd.Start() - if err != nil { - return nil, err - } - _, err = io.WriteString(stdin, registryHost) - if err != nil { - return nil, err - } - err = stdin.Close() - if err != nil { - return nil, err - } - err = cmd.Wait() - if err != nil { - return nil, fmt.Errorf("exec wait error: %w", err) - } - - var config CredentialHelperInput - err = json.Unmarshal([]byte(out.String()), &config) - if err != nil { - return nil, err - } - - return &config, nil -} - -func DockerCredentialBinary(credsStore string) string { - return "docker-credential-" + credsStore -} diff --git a/pkg/docker/login.go b/pkg/docker/login.go index e17f2e123c..c9f5b18706 100644 --- a/pkg/docker/login.go +++ b/pkg/docker/login.go @@ -37,7 +37,7 @@ func saveAuthToConfig(conf *configfile.ConfigFile, registryHost string, username } func saveAuthToCredentialsStore(ctx context.Context, credsStore string, registryHost string, username string, token string) error { - binary := DockerCredentialBinary(credsStore) + binary := dockerCredentialBinary(credsStore) input := CredentialHelperInput{ Username: username, Secret: token, From e23da3af3bda7427376c17b6c74f1de195f2f051 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 11:44:00 -0600 Subject: [PATCH 12/43] move Apt & Tar funcs off command --- pkg/docker/api_client.go | 7 - pkg/docker/apt.go | 81 ----------- pkg/docker/command/command.go | 2 - pkg/docker/docker_command.go | 13 -- pkg/docker/fast_push.go | 4 +- pkg/docker/monobase.go | 137 +++++++++++++++++++ pkg/docker/{apt_test.go => monobase_test.go} | 0 7 files changed, 139 insertions(+), 105 deletions(-) delete mode 100644 pkg/docker/apt.go create mode 100644 pkg/docker/monobase.go rename pkg/docker/{apt_test.go => monobase_test.go} (100%) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 7c26b6b43f..98651bfe4d 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -219,14 +219,7 @@ func (c *apiClient) Push(ctx context.Context, imageRef string) error { // TODO[md]: this doesn't need to be on the interface, move to auth handler func (c *apiClient) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== APIClient.LoadUserInformation %s", registryHost) -} -func (c *apiClient) CreateTarFile(ctx context.Context, ref string, tmpDir string, tarFile string, folder string) (string, error) { - panic("not implemented") -} - -func (c *apiClient) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) { - panic("not implemented") return loadUserInformation(ctx, registryHost) } diff --git a/pkg/docker/apt.go b/pkg/docker/apt.go deleted file mode 100644 index 10a69643ff..0000000000 --- a/pkg/docker/apt.go +++ /dev/null @@ -1,81 +0,0 @@ -package docker - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/replicate/cog/pkg/docker/command" -) - -const aptTarballPrefix = "apt." -const aptTarballSuffix = ".tar.zst" - -func CreateAptTarball(ctx context.Context, tmpDir string, dockerCommand command.Command, packages ...string) (string, error) { - if len(packages) > 0 { - sort.Strings(packages) - hash := sha256.New() - hash.Write([]byte(strings.Join(packages, " "))) - hexHash := hex.EncodeToString(hash.Sum(nil)) - aptTarFile := aptTarballPrefix + hexHash + aptTarballSuffix - aptTarPath := filepath.Join(tmpDir, aptTarFile) - - if _, err := os.Stat(aptTarPath); errors.Is(err, os.ErrNotExist) { - // Remove previous apt tar files. - err = removeAptTarballs(tmpDir) - if err != nil { - return "", err - } - - // Create the apt tar file - _, err = dockerCommand.CreateAptTarFile(ctx, tmpDir, aptTarFile, packages...) - if err != nil { - return "", err - } - } - - return aptTarFile, nil - } - return "", nil -} - -func CurrentAptTarball(tmpDir string) (string, error) { - files, err := os.ReadDir(tmpDir) - if err != nil { - return "", fmt.Errorf("os read dir error: %w", err) - } - - for _, file := range files { - fileName := file.Name() - if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { - return filepath.Join(tmpDir, fileName), nil - } - } - - return "", nil -} - -func removeAptTarballs(tmpDir string) error { - files, err := os.ReadDir(tmpDir) - if err != nil { - return err - } - - for _, file := range files { - fileName := file.Name() - if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { - err = os.Remove(filepath.Join(tmpDir, fileName)) - if err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index c2a67a86a6..f397fc7485 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -15,8 +15,6 @@ type Command interface { Pull(ctx context.Context, ref string, force bool) (*image.InspectResponse, error) Push(ctx context.Context, ref string) error LoadUserInformation(ctx context.Context, registryHost string) (*UserInfo, error) - CreateTarFile(ctx context.Context, ref string, tmpDir string, tarFile string, folder string) (string, error) - CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) ImageExists(ctx context.Context, ref string) (bool, error) ContainerLogs(ctx context.Context, containerID string, w io.Writer) error diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 29d10da52c..65eeec4261 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -9,14 +9,10 @@ import ( "io" "os" "os/exec" - "path/filepath" "runtime" "strings" "github.com/creack/pty" - "github.com/docker/cli/cli/config" - "github.com/docker/cli/cli/config/configfile" - "github.com/docker/cli/cli/config/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/mattn/go-isatty" @@ -95,15 +91,6 @@ func (c *DockerCommand) Push(ctx context.Context, image string) error { func (c *DockerCommand) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== DockerCommand.LoadUserInformation %s", registryHost) - conf := config.LoadDefaultConfigFile(os.Stderr) -func (c *DockerCommand) CreateTarFile(ctx context.Context, image string, tmpDir string, tarFile string, folder string) (string, error) { - console.Debugf("=== DockerCommand.CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) - - args := []string{ - "run", - "--rm", - tmpDir + ":/buildtmp", - "r8.im/monobase:latest", return loadUserInformation(ctx, registryHost) } diff --git a/pkg/docker/fast_push.go b/pkg/docker/fast_push.go index f67bd92766..b002c57d27 100644 --- a/pkg/docker/fast_push.go +++ b/pkg/docker/fast_push.go @@ -147,11 +147,11 @@ func FastPush(ctx context.Context, image string, projectDir string, command comm } func createPythonPackagesTarFile(ctx context.Context, image string, tmpDir string, command command.Command) (string, error) { - return command.CreateTarFile(ctx, image, tmpDir, requirementsTarFile, "root/.venv") + return CreateTarFile(ctx, command, image, tmpDir, requirementsTarFile, "root/.venv") } func createSrcTarFile(ctx context.Context, image string, tmpDir string, command command.Command) (string, error) { - return command.CreateTarFile(ctx, image, tmpDir, "src.tar.zst", "src") + return CreateTarFile(ctx, command, image, tmpDir, "src.tar.zst", "src") } func createWeightsFilesFromWeightsManifest(weights []weights.Weight) []web.File { diff --git a/pkg/docker/monobase.go b/pkg/docker/monobase.go new file mode 100644 index 0000000000..c3a3e8800f --- /dev/null +++ b/pkg/docker/monobase.go @@ -0,0 +1,137 @@ +package docker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" +) + +const aptTarballPrefix = "apt." +const aptTarballSuffix = ".tar.zst" + +func CreateAptTarball(ctx context.Context, tmpDir string, dockerClient command.Command, packages ...string) (string, error) { + if len(packages) > 0 { + sort.Strings(packages) + hash := sha256.New() + hash.Write([]byte(strings.Join(packages, " "))) + hexHash := hex.EncodeToString(hash.Sum(nil)) + aptTarFile := aptTarballPrefix + hexHash + aptTarballSuffix + aptTarPath := filepath.Join(tmpDir, aptTarFile) + + if _, err := os.Stat(aptTarPath); errors.Is(err, os.ErrNotExist) { + // Remove previous apt tar files. + err = removeAptTarballs(tmpDir) + if err != nil { + return "", err + } + + // Create the apt tar file + _, err = CreateAptTarFile(ctx, dockerClient, tmpDir, aptTarFile, packages...) + if err != nil { + return "", err + } + } + + return aptTarFile, nil + } + return "", nil +} + +func CurrentAptTarball(tmpDir string) (string, error) { + files, err := os.ReadDir(tmpDir) + if err != nil { + return "", fmt.Errorf("os read dir error: %w", err) + } + + for _, file := range files { + fileName := file.Name() + if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { + return filepath.Join(tmpDir, fileName), nil + } + } + + return "", nil +} + +func removeAptTarballs(tmpDir string) error { + files, err := os.ReadDir(tmpDir) + if err != nil { + return err + } + + for _, file := range files { + fileName := file.Name() + if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { + err = os.Remove(filepath.Join(tmpDir, fileName)) + if err != nil { + return err + } + } + } + + return nil +} + +func CreateTarFile(ctx context.Context, dockerClient command.Command, image string, tmpDir string, tarFile string, folder string) (string, error) { + console.Debugf("=== DockerCommand.CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) + + opts := command.RunOptions{ + Image: image, + Args: []string{ + "/opt/r8/monobase/tar.sh", + path.Join("/buildtmp", tarFile), + "/", + folder, + }, + Volumes: []command.Volume{ + { + Source: tmpDir, + Destination: "/buildtmp", + }, + }, + } + + if err := dockerClient.Run(ctx, opts); err != nil { + return "", err + } + + return filepath.Join(tmpDir, tarFile), nil +} + +func CreateAptTarFile(ctx context.Context, dockerClient command.Command, tmpDir string, aptTarFile string, packages ...string) (string, error) { + console.Debugf("=== DockerCommand.CreateAptTarFile %s %s", aptTarFile, packages) + + // This uses a hardcoded monobase image to produce an apt tar file. + // The reason being that this apt tar file is created outside the docker file, and it is created by + // running the apt.sh script on the monobase with the packages we intend to install, which produces + // a tar file that can be untarred into a docker build to achieve the equivalent of an apt-get install. + + opts := command.RunOptions{ + Image: "r8.im/monobase:latest", + Args: []string{ + "/opt/r8/monobase/apt.sh", "/buildtmp/" + aptTarFile, + }, + Volumes: []command.Volume{ + { + Source: tmpDir, + Destination: "/buildtmp", + }, + }, + } + + if err := dockerClient.Run(ctx, opts); err != nil { + return "", err + } + + return aptTarFile, nil +} diff --git a/pkg/docker/apt_test.go b/pkg/docker/monobase_test.go similarity index 100% rename from pkg/docker/apt_test.go rename to pkg/docker/monobase_test.go From 6b8a1da8073e660687b30de6be15e33b793f0cd9 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 11:44:08 -0600 Subject: [PATCH 13/43] lint fixes --- pkg/docker/command/command.go | 1 - pkg/registry_testhelpers/registry_container.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index f397fc7485..aeac04bec6 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -27,7 +27,6 @@ type Command interface { } type ImageBuildOptions struct { - // Platform string WorkingDir string DockerfileContents string ImageName string diff --git a/pkg/registry_testhelpers/registry_container.go b/pkg/registry_testhelpers/registry_container.go index 3e6ec35270..bc0cfb3521 100644 --- a/pkg/registry_testhelpers/registry_container.go +++ b/pkg/registry_testhelpers/registry_container.go @@ -158,5 +158,4 @@ func generateHtpasswd(username, password string) (string, error) { return "", err } return fmt.Sprintf("%s:%s", username, string(hash)), nil - // return fmt.Sprintf("%s:$2y$05$%s", username, base64.StdEncoding.EncodeToString([]byte(password))) } From 0d9e18ed0dc220e25a388c2c4c6478c52c85dfb1 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 12:04:58 -0600 Subject: [PATCH 14/43] hack mockcommand to fake tar scripts for tests --- pkg/docker/dockertest/mock_command.go | 10 ++++++++++ pkg/docker/monobase.go | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/docker/dockertest/mock_command.go b/pkg/docker/dockertest/mock_command.go index e27c715ed2..5b399d9aab 100644 --- a/pkg/docker/dockertest/mock_command.go +++ b/pkg/docker/dockertest/mock_command.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" @@ -99,6 +100,15 @@ func (c *MockCommand) ImageBuild(ctx context.Context, options command.ImageBuild } func (c *MockCommand) Run(ctx context.Context, options command.RunOptions) error { + // hack to handle generating tar files for monobase + if options.Args[0] == "/opt/r8/monobase/tar.sh" || options.Args[0] == "/opt/r8/monobase/apt.sh" { + tmpDir := options.Volumes[0].Source + tarfile := strings.TrimPrefix(options.Args[1], "/buildtmp/") + + outPath := filepath.Join(tmpDir, tarfile) + return os.WriteFile(outPath, []byte("hello\ngo\n"), 0o644) + } + panic("not implemented") } diff --git a/pkg/docker/monobase.go b/pkg/docker/monobase.go index c3a3e8800f..fd14647cb1 100644 --- a/pkg/docker/monobase.go +++ b/pkg/docker/monobase.go @@ -83,7 +83,7 @@ func removeAptTarballs(tmpDir string) error { } func CreateTarFile(ctx context.Context, dockerClient command.Command, image string, tmpDir string, tarFile string, folder string) (string, error) { - console.Debugf("=== DockerCommand.CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) + console.Debugf("=== CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) opts := command.RunOptions{ Image: image, @@ -109,7 +109,7 @@ func CreateTarFile(ctx context.Context, dockerClient command.Command, image stri } func CreateAptTarFile(ctx context.Context, dockerClient command.Command, tmpDir string, aptTarFile string, packages ...string) (string, error) { - console.Debugf("=== DockerCommand.CreateAptTarFile %s %s", aptTarFile, packages) + console.Debugf("=== CreateAptTarFile %s %s", aptTarFile, packages) // This uses a hardcoded monobase image to produce an apt tar file. // The reason being that this apt tar file is created outside the docker file, and it is created by @@ -119,7 +119,7 @@ func CreateAptTarFile(ctx context.Context, dockerClient command.Command, tmpDir opts := command.RunOptions{ Image: "r8.im/monobase:latest", Args: []string{ - "/opt/r8/monobase/apt.sh", "/buildtmp/" + aptTarFile, + "/opt/r8/monobase/apt.sh", path.Join("/buildtmp", aptTarFile), }, Volumes: []command.Volume{ { From 238d053fc5971822bb41e4f83d42d71c584cf047 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 12:59:28 -0600 Subject: [PATCH 15/43] fix for apt packages not passing to monobase apt.sh --- pkg/docker/monobase.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/docker/monobase.go b/pkg/docker/monobase.go index fd14647cb1..f604a65359 100644 --- a/pkg/docker/monobase.go +++ b/pkg/docker/monobase.go @@ -118,9 +118,13 @@ func CreateAptTarFile(ctx context.Context, dockerClient command.Command, tmpDir opts := command.RunOptions{ Image: "r8.im/monobase:latest", - Args: []string{ - "/opt/r8/monobase/apt.sh", path.Join("/buildtmp", aptTarFile), - }, + Args: append( + []string{ + "/opt/r8/monobase/apt.sh", + path.Join("/buildtmp", aptTarFile), + }, + packages..., + ), Volumes: []command.Volume{ { Source: tmpDir, From 1255a5ff71ca6bdeb1a206a32796a902081d9b6c Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 16:52:56 -0600 Subject: [PATCH 16/43] return new client if env var present --- pkg/docker/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index 7f10395c15..2b61861d63 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -13,7 +13,7 @@ func NewClient(ctx context.Context, opts ...Option) (command.Command, error) { enabled, _ := strconv.ParseBool(os.Getenv("COG_DOCKER_SDK_CLIENT")) if enabled { console.Debugf("Docker client: sdk") - panic("not implemented in this branch :sad-panda:") + return NewAPIClient(ctx, opts...) } console.Debugf("Docker client: cli") From 0d597ac5d9b3713c4e9ab14caf7602aa2f1066c1 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 16:55:16 -0600 Subject: [PATCH 17/43] import buildkit client, add ImageBuild to api client --- go.mod | 31 +++++- go.sum | 90 +++++++++++++++++- pkg/docker/api_client.go | 58 +++++++++++- pkg/docker/buildkit.go | 171 ++++++++++++++++++++++++++++++++++ pkg/docker/command/command.go | 21 +++-- pkg/docker/credentials.go | 62 ++++++++++++ 6 files changed, 414 insertions(+), 19 deletions(-) create mode 100644 pkg/docker/buildkit.go diff --git a/go.mod b/go.mod index 8de9bdc470..35559838fd 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/go-homedir v1.1.0 + github.com/moby/buildkit v0.21.1 github.com/moby/term v0.5.2 github.com/replicate/go v0.0.0-20250205165008-b772d7cd506b github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 @@ -29,6 +30,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 github.com/xeonx/timeago v1.0.0-rc5 + golang.org/x/crypto v0.37.0 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 golang.org/x/sync v0.13.0 golang.org/x/sys v0.33.0 @@ -44,6 +46,7 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/4meepo/tagalign v1.4.2 // indirect github.com/Abirdcfly/dupword v0.1.3 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Antonboom/errname v1.0.0 // indirect github.com/Antonboom/nilnil v1.0.1 // indirect github.com/Antonboom/testifylint v1.5.2 // indirect @@ -89,9 +92,17 @@ require ( github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/ckaznocha/intrange v0.3.0 // indirect + github.com/containerd/console v1.0.4 // indirect + github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/containerd/v2 v2.0.4 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/platforms v1.0.0-rc.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/daixiang0/gci v0.13.5 // indirect @@ -130,6 +141,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect github.com/golangci/go-printf-func-name v0.1.0 // indirect github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect @@ -147,9 +159,12 @@ require ( github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect + github.com/in-toto/in-toto-golang v0.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.3.1 // indirect github.com/jgautheron/goconst v1.7.1 // indirect @@ -182,9 +197,11 @@ require ( github.com/mgechev/revive v1.7.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/signal v0.7.1 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect @@ -201,6 +218,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.7.1 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -223,7 +241,9 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/usestdlibvars v1.28.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect github.com/securego/gosec/v2 v2.22.2 // indirect + github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sivchari/containedctx v1.0.3 // indirect @@ -246,6 +266,10 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/tomarrell/wrapcheck/v2 v2.10.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect + github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583 // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect + github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect + github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/uudashr/gocognit v1.2.0 // indirect @@ -262,6 +286,8 @@ require ( go-simpler.org/musttag v0.13.0 // indirect go-simpler.org/sloglint v0.9.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect @@ -272,13 +298,14 @@ require ( go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.39.0 // indirect golang.org/x/text v0.24.0 // indirect + golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.32.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 // indirect google.golang.org/grpc v1.71.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gotest.tools/gotestsum v1.12.2 // indirect diff --git a/go.sum b/go.sum index 287ef883b7..11f3d3889d 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Abirdcfly/dupword v0.1.3 h1:9Pa1NuAsZvpFPi9Pqkd93I7LIYRURj+A//dFd5tgB github.com/Abirdcfly/dupword v0.1.3/go.mod h1:8VbB2t7e10KRNdwTVoxdBaxla6avbhGzb8sCTygUMhw= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw= github.com/Antonboom/errname v1.0.0 h1:oJOOWR07vS1kRusl6YRSlat7HFnb3mSfMl6sDMRoTBA= github.com/Antonboom/errname v1.0.0/go.mod h1:gMOBFzK/vrTiXN9Oh+HFs+e6Ndl0eTFbtsRTSRdXyGI= github.com/Antonboom/nilnil v1.0.1 h1:C3Tkm0KUxgfO4Duk3PM+ztPncTFlOf0b2qadmS0s4xs= @@ -30,6 +32,8 @@ github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7r github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.12.9 h1:2zJy5KA+l0loz1HzEGqyNnjd3fyZA31ZBCGKacp6lLg= +github.com/Microsoft/hcsshim v0.12.9/go.mod h1:fJ0gkFAna6ukt0bLdKB8djt4XIJhF/vEPuoIWYVvZ8Y= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= @@ -52,6 +56,8 @@ github.com/alingse/nilnesserr v0.1.2 h1:Yf8Iwm3z2hUUrP4muWfW83DF4nE3r1xZ26fGWUKC github.com/alingse/nilnesserr v0.1.2/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.2.0 h1:/2Lp1bypdmK9wDIq7uWBlDF1iMUpIIS4A+pF6C9IEUU= @@ -112,12 +118,38 @@ github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww= github.com/ckaznocha/intrange v0.3.0 h1:VqnxtK32pxgkhJgYQEeOArVidIPg+ahLP7WBOXZd5ZY= github.com/ckaznocha/intrange v0.3.0/go.mod h1:+I/o2d2A1FBHgGELbGxzIcyd3/9l9DuwjM8FsbSS3Lo= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= +github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= +github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/containerd/v2 v2.0.4 h1:+r7yJMwhTfMm3CDyiBjMBQO8a9CTBxL2Bg/JtqtIwB8= +github.com/containerd/containerd/v2 v2.0.4/go.mod h1:5j9QUUaV/cy9ZeAx4S+8n9ffpf+iYnEj4jiExgcbuLY= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/containerd/nydus-snapshotter v0.15.0 h1:RqZRs1GPeM6T3wmuxJV9u+2Rg4YETVMwTmiDeX+iWC8= +github.com/containerd/nydus-snapshotter v0.15.0/go.mod h1:biq0ijpeZe0I5yZFSJyHzFSjjRZQ7P7y/OuHyd7hYOw= +github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= +github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= +github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y= +github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8= github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -221,6 +253,10 @@ github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= github.com/golangci/go-printf-func-name v0.1.0 h1:dVokQP+NMTO7jwO4bwsRwLWeudOVUPPyAKJuzv8pEJU= @@ -270,8 +306,13 @@ github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+ github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -281,6 +322,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY= +github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1GdHMCq8+WPxw8/BE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= @@ -356,16 +399,24 @@ github.com/mgechev/revive v1.7.0 h1:JyeQ4yO5K8aZhIKf5rec56u0376h8AlKNQEmjfkjKlY= github.com/mgechev/revive v1.7.0/go.mod h1:qZnwcNhoguE58dfi96IJeSTPeZQejNeoMQLUZGi4SW4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/buildkit v0.21.1 h1:wTjVLfirh7skZt9piaIlNo8WdiPjza1CDl2EArDV9bA= +github.com/moby/buildkit v0.21.1/go.mod h1:mBq0D44uCyz2PdX8T/qym5LBbkBO3GGv0wqgX9ABYYw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -398,6 +449,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= +github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= @@ -405,12 +460,16 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -464,8 +523,12 @@ github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tM github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.28.0 h1:jZnudE2zKCtYlGzLVreNp5pmCdOxXUzwsMDBkR21cyQ= github.com/sashamelentyev/usestdlibvars v1.28.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= +github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= +github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= github.com/securego/gosec/v2 v2.22.2 h1:IXbuI7cJninj0nRpZSLCUlotsj8jGusohfONMrHoF6g= github.com/securego/gosec/v2 v2.22.2/go.mod h1:UEBGA+dSKb+VqM6TdehR7lnQtIIMorYJ4/9CW1KVQBE= +github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= +github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil/v4 v4.25.2 h1:NMscG3l2CqtWFS86kj3vP7soOczqrQYIEhO/pMvvQkk= github.com/shirou/gopsutil/v4 v4.25.2/go.mod h1:34gBYJzyqCDT11b6bMHP0XCvWeU3J61XRT7a2EmCRTA= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -482,6 +545,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY= +github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= @@ -538,6 +603,14 @@ github.com/tomarrell/wrapcheck/v2 v2.10.0 h1:SzRCryzy4IrAH7bVGG4cK40tNUhmVmMDuJu github.com/tomarrell/wrapcheck/v2 v2.10.0/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583 h1:mK+ZskNt7SG4dxfKIi27C7qHAQzyjAVt1iyTf0hmsNc= +github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= +github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= +github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= @@ -588,8 +661,14 @@ go-simpler.org/musttag v0.13.0 h1:Q/YAW0AHvaoaIbsPj3bvEI5/QFP7w696IMUpnKXQfCE= go-simpler.org/musttag v0.13.0/go.mod h1:FTzIGeK6OkKlUDVpj0iQUXZLUO1Js9+mvykDQy9C5yM= go-simpler.org/sloglint v0.9.0 h1:/40NQtjRx9txvsB/RN022KsUJU+zaaSb/9q9BSefSrE= go-simpler.org/sloglint v0.9.0/go.mod h1:G/OrAF6uxj48sHahCzrbarVMptL2kjWTaUeC8+fOGww= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= @@ -602,6 +681,8 @@ go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/ go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= @@ -688,6 +769,7 @@ golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -719,8 +801,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 98651bfe4d..296ecd7ea5 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "os" "github.com/docker/docker/api/types/container" @@ -15,6 +16,9 @@ import ( "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" "github.com/google/go-containerregistry/pkg/name" + buildkitclient "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "golang.org/x/sync/errgroup" "github.com/replicate/go/types/ptr" @@ -255,7 +259,55 @@ func (c *apiClient) ImageExists(ctx context.Context, ref string) (bool, error) { } func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOptions) error { - panic("not implemented") + buildDir, err := os.MkdirTemp("", "cog-build") + if err != nil { + return err + } + defer os.RemoveAll(buildDir) + + bc, err := buildkitclient.New(ctx, "", + // Connect to Docker Engine's embedded Buildkit. + buildkitclient.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return c.client.DialHijack(ctx, "/grpc", "h2c", map[string][]string{}) + }), + ) + if err != nil { + return err + } + + statusCh := make(chan *buildkitclient.SolveStatus) + var res *buildkitclient.SolveResponse + + // Build the image. + eg, ctx := errgroup.WithContext(ctx) + + // run the display in a goroutine + eg.Go(newDisplay(statusCh)) + + // run the build in a goroutine + eg.Go(func() error { + options, err := solveOptFromImageOptions(buildDir, options) + if err != nil { + return err + } + + res, err = bc.Solve(ctx, nil, options, statusCh) + if err != nil { + return err + } + return nil + }) + err = eg.Wait() + + if err != nil { + return err + } + + console.Debugf("image digest %s", res.ExporterResponse[exptypes.ExporterImageDigestKey]) + + // TODO[md]: return the image id on success + // return res.ExporterResponse[exptypes.ExporterImageDigestKey], nil + return nil } func (c *apiClient) Run(ctx context.Context, options command.RunOptions) error { @@ -265,7 +317,3 @@ func (c *apiClient) Run(ctx context.Context, options command.RunOptions) error { func (c *apiClient) ContainerStart(ctx context.Context, options command.RunOptions) (string, error) { panic("not implemented") } - -func (c *apiClient) ContainerRemove(ctx context.Context, containerID string) error { - panic("not implemented") -} diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go new file mode 100644 index 0000000000..767121f2ce --- /dev/null +++ b/pkg/docker/buildkit.go @@ -0,0 +1,171 @@ +package docker + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/docker/docker/api/types/registry" + buildkitclient "github.com/moby/buildkit/client" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/auth" + "github.com/moby/buildkit/session/secrets/secretsprovider" + "github.com/moby/buildkit/util/progress/progressui" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/replicate/cog/pkg/docker/command" +) + +func prepareDockerfileDir(buildDir string, dockerfileContents string) (string, error) { + dockerfilePath := filepath.Join(buildDir, "Dockerfile") + err := os.WriteFile(dockerfilePath, []byte(dockerfileContents), 0o644) + if err != nil { + return "", err + } + return dockerfilePath, nil +} + +func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) (buildkitclient.SolveOpt, error) { + dockerfilePath, err := prepareDockerfileDir(buildDir, opts.DockerfileContents) + if err != nil { + return buildkitclient.SolveOpt{}, err + } + + // first, configure the frontend, in this case, dockerfile.v0 + frontendAttrs := map[string]string{ + // filename is the path to the Dockerfile within the "dockerfile" LocalDir context + "filename": filepath.Base(dockerfilePath), + // target is the name of a stage in a multi-stage Dockerfile + // "target": opts.Target, + // Replicate only supports linux/amd64, but local Docker Engine could be running on ARM, + // including Apple Silicon. Force it to linux/amd64 for now. + "platform": "linux/amd64", + } + + // disable cache if requested + if opts.NoCache { + frontendAttrs["no-cache"] = "" + } + + // add labels to the image + for k, v := range opts.Labels { + frontendAttrs["label:"+k] = v + } + + // add build args to the image + for k, v := range opts.BuildArgs { + if v == nil { + continue + } + frontendAttrs["build-arg:"+k] = *v + } + + solveOpts := buildkitclient.SolveOpt{ + Frontend: "dockerfile.v0", + FrontendAttrs: frontendAttrs, + LocalDirs: map[string]string{ + "dockerfile": filepath.Dir(dockerfilePath), + "context": opts.WorkingDir, + }, + // Docker Engine's worker only supports three exporters. + // "moby" exporter works best for cog, since we want to keep images in + // Docker Engine's image store. The others are exporting images to somewhere else. + // https://github.com/moby/moby/blob/v20.10.24/builder/builder-next/worker/worker.go#L221 + Exports: []buildkitclient.ExportEntry{ + {Type: "moby", Attrs: map[string]string{"name": opts.ImageName}}, + }, + } + + // add auth provider to the session so the local engine can pull and push images + solveOpts.Session = append( + solveOpts.Session, + newBuildkitAuthProvider("r8.im"), + ) + + // add secrets to the session + if len(opts.BuildSecrets) > 0 { + secrets := make(map[string][]byte) + for k, v := range opts.BuildSecrets { + secrets[k] = []byte(v) + } + + solveOpts.Session = append( + solveOpts.Session, + secretsprovider.FromMap(secrets), + ) + } + + return solveOpts, nil +} + +func newDisplay(statusCh chan *buildkitclient.SolveStatus) func() error { + return func() error { + display, err := progressui.NewDisplay( + os.Stderr, + progressui.DisplayMode(os.Getenv("BUILDKIT_PROGRESS")), + // progressui.WithPhase("BUILDINGGGGG"), + // progressui.WithDesc("SOMETEXT", "SOMECONSOLE"), + ) + if err != nil { + return err + } + + // UpdateFrom must not use the incoming context. + // Canceling this context kills the reader of statusCh which blocks buildkit.Client's Solve() indefinitely. + // Solve() closes statusCh at the end and UpdateFrom returns by reading the closed channel. + // + // See https://github.com/superfly/flyctl/pull/2682 for the context. + _, err = display.UpdateFrom(context.Background(), statusCh) + return err + + } +} + +func newBuildkitAuthProvider(registryHosts ...string) session.Attachable { + return &buildkitAuthProvider{ + registryHosts: sync.OnceValues(func() (map[string]registry.AuthConfig, error) { + return loadRegistryAuths(context.Background(), registryHosts...) + }), + // token: token, + } +} + +type buildkitAuthProvider struct { + registryHosts func() (map[string]registry.AuthConfig, error) + // auths map[string]registry.AuthConfig + // token string +} + +func (ap *buildkitAuthProvider) Register(server *grpc.Server) { + auth.RegisterAuthServer(server, ap) +} + +func (ap *buildkitAuthProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) { + auths, err := ap.registryHosts() + if err != nil { + return nil, fmt.Errorf("failed to load registry auth configs: %w", err) + } + res := &auth.CredentialsResponse{} + if a, ok := auths[req.Host]; ok { + res.Username = a.Username + res.Secret = a.Password + } + + return res, nil +} + +func (ap *buildkitAuthProvider) FetchToken(ctx context.Context, req *auth.FetchTokenRequest) (*auth.FetchTokenResponse, error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens disabled") +} + +func (ap *buildkitAuthProvider) GetTokenAuthority(ctx context.Context, req *auth.GetTokenAuthorityRequest) (*auth.GetTokenAuthorityResponse, error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens disabled") +} + +func (ap *buildkitAuthProvider) VerifyTokenAuthority(ctx context.Context, req *auth.VerifyTokenAuthorityRequest) (*auth.VerifyTokenAuthorityResponse, error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens disabled") +} diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index aeac04bec6..af28faa6ac 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -29,14 +29,19 @@ type Command interface { type ImageBuildOptions struct { WorkingDir string DockerfileContents string - ImageName string - Secrets []string - NoCache bool - ProgressOutput string - Epoch *int64 - ContextDir string - BuildContexts map[string]string - Labels map[string]string + // TODO[md]: ImageName should be renamed to Tag + ImageName string + Secrets []string + NoCache bool + ProgressOutput string + Epoch *int64 + ContextDir string + BuildContexts map[string]string + Labels map[string]string + + // only supported on buildkit client, not cli client + BuildSecrets map[string]string + BuildArgs map[string]*string } type RunOptions struct { diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go index 28b26332b9..722dc99851 100644 --- a/pkg/docker/credentials.go +++ b/pkg/docker/credentials.go @@ -12,6 +12,7 @@ import ( "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/config/types" + "github.com/docker/docker/api/types/registry" "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/util/console" @@ -44,6 +45,67 @@ func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types return conf.AuthConfigs[registryHost], nil } +func loadRegistryAuths(ctx context.Context, registryHosts ...string) (map[string]registry.AuthConfig, error) { + conf := config.LoadDefaultConfigFile(os.Stderr) + + out := make(map[string]registry.AuthConfig) + + for _, host := range registryHosts { + console.Debugf("=== loadRegistryAuths %s", host) + // check the credentials store first if set + if conf.CredentialsStore != "" { + console.Debugf("=== loadRegistryAuths %s: credentials store set", host) + credsHelper, err := loadAuthFromCredentialsStore(ctx, conf.CredentialsStore, host) + if err != nil { + console.Debugf("=== loadRegistryAuths %s: error loading credentials store: %s", host, err) + return nil, err + } + console.Debugf("=== loadRegistryAuths %s: credentials store loaded", host) + out[host] = registry.AuthConfig{ + Username: credsHelper.Username, + Password: credsHelper.Secret, + ServerAddress: host, + } + continue + } + + // next, check if the auth config exists in the config file + if auth, ok := conf.AuthConfigs[host]; ok { + console.Debugf("=== loadRegistryAuths %s: auth config found in config file", host) + out[host] = registry.AuthConfig{ + Username: auth.Username, + Password: auth.Password, + Auth: auth.Auth, + Email: auth.Email, + ServerAddress: host, + IdentityToken: auth.IdentityToken, + RegistryToken: auth.RegistryToken, + } + continue + } + + console.Debugf("=== loadRegistryAuths %s: no auth config found", host) + + // TODO[md]: should we error here!? probably not... + } + + return out, nil +} + +// func loadRegistryAuths(conf *configfile.ConfigFile, registryHosts ...string) (map[string]types.AuthConfig, error) { +// out := make(map[string]types.AuthConfig) +// for _, host := range registryHosts { +// auth, ok := conf.AuthConfigs[host] +// if !ok { +// // TODO[md]: should we return an error here or just carry on? +// return nil, fmt.Errorf("no auth config found for registry host %s", host) +// } +// out[host] = auth +// } + +// return out, nil +// } + func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { var out strings.Builder binary := dockerCredentialBinary(credsStore) From bb04bef0f860ce925af81e7cc4fc85c9afe75617 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 20:53:23 -0600 Subject: [PATCH 18/43] container run, start, stop --- pkg/docker/api_client.go | 240 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 2 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 296ecd7ea5..2f1a9127b2 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -7,17 +7,24 @@ import ( "io" "net" "os" + "strconv" + "strings" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" dc "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-connections/nat" "github.com/google/go-containerregistry/pkg/name" + "github.com/mattn/go-isatty" buildkitclient "github.com/moby/buildkit/client" "github.com/moby/buildkit/exporter/containerimage/exptypes" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "github.com/replicate/go/types/ptr" @@ -259,6 +266,8 @@ func (c *apiClient) ImageExists(ctx context.Context, ref string) (bool, error) { } func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOptions) error { + console.Debugf("=== APIClient.ImageBuild %s", options.ImageName) + buildDir, err := os.MkdirTemp("", "cog-build") if err != nil { return err @@ -310,10 +319,237 @@ func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOp return nil } +func (c *apiClient) containerRun(ctx context.Context, options command.RunOptions) (string, error) { + console.Debugf("=== APIClient.containerRun %s", options.Image) + + containerCfg := &container.Config{ + Image: options.Image, + Cmd: options.Args, + Env: options.Env, + AttachStdin: options.Stdin != nil, + AttachStdout: options.Stdout != nil, + AttachStderr: options.Stderr != nil, + Tty: false, // Will be set below if stdin is a TTY + } + + // Set working directory if specified + if options.Workdir != "" { + containerCfg.WorkingDir = options.Workdir + } + + // Check if stdin is a TTY + if options.Stdin != nil { + if f, ok := options.Stdin.(*os.File); ok { + containerCfg.Tty = isatty.IsTerminal(f.Fd()) + } + } + + hostCfg := &container.HostConfig{ + // always remove container after it exits + // AutoRemove: true, + // https://github.com/pytorch/pytorch/issues/2244 + // https://github.com/replicate/cog/issues/1293 + ShmSize: 6 * 1024 * 1024 * 1024, // 6GB + Resources: container.Resources{}, + } + + if options.GPUs != "" { + deviceRequest, err := parseGPURequest(options) + if err != nil { + return "", err + } + hostCfg.Resources.DeviceRequests = []container.DeviceRequest{deviceRequest} + } + + // Configure port bindings + if len(options.Ports) > 0 { + hostCfg.PortBindings = make(nat.PortMap) + for _, port := range options.Ports { + containerPort := nat.Port(fmt.Sprintf("%d/tcp", port.ContainerPort)) + hostCfg.PortBindings[containerPort] = []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: strconv.Itoa(port.HostPort), + }, + } + } + } + + // Configure volume bindings + if len(options.Volumes) > 0 { + hostCfg.Binds = make([]string, len(options.Volumes)) + for i, volume := range options.Volumes { + hostCfg.Binds[i] = fmt.Sprintf("%s:%s", volume.Source, volume.Destination) + } + } + + networkingCfg := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + // "bridge": {}, + }, + } + + platform := &ocispec.Platform{ + // force platform to linux/amd64 + Architecture: "amd64", + OS: "linux", + } + + runContainer, err := c.client.ContainerCreate(ctx, + containerCfg, + hostCfg, + networkingCfg, + platform, + "") + if err != nil { + return "", fmt.Errorf("failed to create container: %w", err) + } + // make sure the container is removed if it fails to start + // defer func() { + // if err := c.client.ContainerRemove(ctx, runContainer.ID, container.RemoveOptions{ + // RemoveVolumes: true, + // RemoveLinks: false, + // Force: true, + // }); err != nil { + // console.Warnf("failed to remove container: %s", err) + // } + // }() + + console.Debugf("container id: %s", runContainer.ID) + + // Create error group for stream copying + var eg *errgroup.Group + var stream types.HijackedResponse + + // Attach to container streams if we have any writers and not detached + if !options.Detach && (options.Stdout != nil || options.Stderr != nil || options.Stdin != nil) { + attachOpts := container.AttachOptions{ + Stream: true, + Stdin: options.Stdin != nil, + Stdout: options.Stdout != nil, + Stderr: options.Stderr != nil, + } + + var err error + stream, err = c.client.ContainerAttach(ctx, runContainer.ID, attachOpts) + if err != nil { + return "", fmt.Errorf("failed to attach to container: %w", err) + } + defer stream.Close() + + // Start copying streams in the background + eg, _ = errgroup.WithContext(ctx) + if options.Stdout != nil || options.Stderr != nil { + eg.Go(func() error { + if containerCfg.Tty { + _, err = io.Copy(options.Stdout, stream.Reader) + } else { + _, err = stdcopy.StdCopy(options.Stdout, options.Stderr, stream.Reader) + } + return err + }) + } + if options.Stdin != nil { + eg.Go(func() error { + _, err = io.Copy(stream.Conn, options.Stdin) + return err + }) + } + } + + // Start the container + if err := c.client.ContainerStart(ctx, runContainer.ID, container.StartOptions{}); err != nil { + return "", fmt.Errorf("failed to start container: %w", err) + } + + // If detached, wait for container to be running before returning + if options.Detach { + // // Wait for container to be running + // statusCh, errCh := c.client.ContainerWait(ctx, runContainer.ID, container.WaitConditionNextExit) + // select { + // case err := <-errCh: + // return "", fmt.Errorf("error waiting for container to start: %w", err) + // case status := <-statusCh: + // if status.StatusCode != 0 { + // return "", fmt.Errorf("container failed to start with status %d", status.StatusCode) + // } + // } + return runContainer.ID, nil + } + + // Wait for the container to exit + statusCh, errCh := c.client.ContainerWait(ctx, runContainer.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + return "", fmt.Errorf("error waiting for container: %w", err) + case status := <-statusCh: + if status.StatusCode != 0 { + return "", fmt.Errorf("container exited with status %d", status.StatusCode) + } + } + + // Wait for stream copying to complete + if eg != nil { + if err := eg.Wait(); err != nil { + return "", fmt.Errorf("error copying streams: %w", err) + } + } + + return runContainer.ID, nil +} + func (c *apiClient) Run(ctx context.Context, options command.RunOptions) error { - panic("not implemented") + console.Debugf("=== APIClient.Run %s", options.Image) + + if options.Stdout == nil { + options.Stdout = os.Stdout + } + if options.Stderr == nil { + options.Stderr = os.Stderr + } + + _, err := c.containerRun(ctx, options) + return err } func (c *apiClient) ContainerStart(ctx context.Context, options command.RunOptions) (string, error) { - panic("not implemented") + console.Debugf("=== APIClient.ContainerStart %s", options.Image) + + options.Detach = true + fmt.Println("container run") + id, err := c.containerRun(ctx, options) + fmt.Println("container run AFTER", id, err) + return id, err +} + +// parseGPURequest converts a Docker CLI --gpus string into a DeviceRequest slice +func parseGPURequest(opts command.RunOptions) (container.DeviceRequest, error) { + if opts.GPUs == "" { + return container.DeviceRequest{}, nil + } + + deviceRequest := container.DeviceRequest{ + Driver: "nvidia", + Capabilities: [][]string{{"gpu"}}, + } + + // Parse the GPUs string + switch opts.GPUs { + case "all": + deviceRequest.Count = -1 // Use all available GPUs + default: + // Check if it's a number + if count, err := strconv.Atoi(opts.GPUs); err == nil { + deviceRequest.Count = count + } else if strings.HasPrefix(opts.GPUs, "device=") { + // Handle device=0,1 format + devices := strings.TrimPrefix(opts.GPUs, "device=") + deviceRequest.DeviceIDs = strings.Split(devices, ",") + } else { + // Invalid GPU specification, return nil to indicate no GPU access + return container.DeviceRequest{}, fmt.Errorf("invalid GPU specification: %q", opts.GPUs) + } + } + + return deviceRequest, nil } From 25763174ad9c362a0dfae0ec47bea7a553d05a6c Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 20:53:52 -0600 Subject: [PATCH 19/43] run integration tests for both docker clients --- .github/workflows/ci.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 14dde2c4bd..96602aadbb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -63,6 +63,7 @@ jobs: matrix: # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources platform: [ubuntu-latest, macos-latest] + new_docker_api_client: [true, false] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v4 @@ -82,6 +83,8 @@ jobs: run: make cog - name: Test run: make test-go + env: + COG_DOCKER_SDK_CLIENT: ${{ matrix.new_docker_api_client }} test-python: name: "Test Python ${{ matrix.python-version }} + Pydantic v${{ matrix.pydantic }}" @@ -121,6 +124,9 @@ jobs: name: "Test integration" needs: build-python runs-on: ubuntu-latest-16-cores + strategy: + matrix: + new_docker_api_client: [true, false] timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -146,6 +152,7 @@ jobs: run: make test-integration env: R8_COGLET_VERSION: coglet==0.1.0-alpha17 + COG_DOCKER_SDK_CLIENT: ${{ matrix.new_docker_api_client }} release: name: "Release" From a8d36cc476dba61d88257c2f724925090ccf9566 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 21:01:54 -0600 Subject: [PATCH 20/43] stop failing fast on integration tests --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 96602aadbb..c144b6998a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,6 +125,7 @@ jobs: needs: build-python runs-on: ubuntu-latest-16-cores strategy: + fail-fast: false matrix: new_docker_api_client: [true, false] timeout-minutes: 20 From a670b562788fbad911e8c4ba792bb282b68edeca Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Thu, 15 May 2025 21:24:26 -0600 Subject: [PATCH 21/43] lint fixes --- pkg/docker/api_client.go | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 2f1a9127b2..673c492e2f 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -238,7 +238,7 @@ func (c *apiClient) Inspect(ctx context.Context, ref string) (*image.InspectResp console.Debugf("=== APIClient.Inspect %s", ref) // TODO[md]: platform requires engine 1.49+, and it's not widly available as of 2025-05. - // platform := ocispec.Platform{OS: "linux", Architecture: "amd64"} + // platform := ocispec.Platform{OS: "linux", Architecture: "amd64"} // client.ImageInspectWithPlatform(&platform), inspect, err := c.client.ImageInspect(ctx, ref) @@ -315,7 +315,6 @@ func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOp console.Debugf("image digest %s", res.ExporterResponse[exptypes.ExporterImageDigestKey]) // TODO[md]: return the image id on success - // return res.ExporterResponse[exptypes.ExporterImageDigestKey], nil return nil } @@ -384,9 +383,7 @@ func (c *apiClient) containerRun(ctx context.Context, options command.RunOptions } networkingCfg := &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - // "bridge": {}, - }, + EndpointsConfig: map[string]*network.EndpointSettings{}, } platform := &ocispec.Platform{ @@ -404,16 +401,7 @@ func (c *apiClient) containerRun(ctx context.Context, options command.RunOptions if err != nil { return "", fmt.Errorf("failed to create container: %w", err) } - // make sure the container is removed if it fails to start - // defer func() { - // if err := c.client.ContainerRemove(ctx, runContainer.ID, container.RemoveOptions{ - // RemoveVolumes: true, - // RemoveLinks: false, - // Force: true, - // }); err != nil { - // console.Warnf("failed to remove container: %s", err) - // } - // }() + // TODO[md]: ensure the container is removed if start & auto-remove fails console.Debugf("container id: %s", runContainer.ID) @@ -464,16 +452,6 @@ func (c *apiClient) containerRun(ctx context.Context, options command.RunOptions // If detached, wait for container to be running before returning if options.Detach { - // // Wait for container to be running - // statusCh, errCh := c.client.ContainerWait(ctx, runContainer.ID, container.WaitConditionNextExit) - // select { - // case err := <-errCh: - // return "", fmt.Errorf("error waiting for container to start: %w", err) - // case status := <-statusCh: - // if status.StatusCode != 0 { - // return "", fmt.Errorf("container failed to start with status %d", status.StatusCode) - // } - // } return runContainer.ID, nil } From 633cf03541bf0e21143542ffd6385c7c6c164bd8 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 11:13:49 -0600 Subject: [PATCH 22/43] handle cache, contexts, timestamps, and epoch build opts --- pkg/docker/api_client.go | 13 +++++++-- pkg/docker/buildkit.go | 62 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index 673c492e2f..bb4f2d3a1c 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -287,11 +287,20 @@ func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOp statusCh := make(chan *buildkitclient.SolveStatus) var res *buildkitclient.SolveResponse + // Determine display mode: options.ProgressOutput > env > 'auto' + displayMode := options.ProgressOutput + if displayMode == "" { + displayMode = os.Getenv("BUILDKIT_PROGRESS") + } + if displayMode == "" { + displayMode = "auto" + } + // Build the image. eg, ctx := errgroup.WithContext(ctx) // run the display in a goroutine - eg.Go(newDisplay(statusCh)) + eg.Go(newDisplay(statusCh, displayMode)) // run the build in a goroutine eg.Go(func() error { @@ -494,9 +503,7 @@ func (c *apiClient) ContainerStart(ctx context.Context, options command.RunOptio console.Debugf("=== APIClient.ContainerStart %s", options.Image) options.Detach = true - fmt.Println("container run") id, err := c.containerRun(ctx, options) - fmt.Println("container run AFTER", id, err) return id, err } diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go index 767121f2ce..87a1210fdb 100644 --- a/pkg/docker/buildkit.go +++ b/pkg/docker/buildkit.go @@ -17,7 +17,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + cogconfig "github.com/replicate/cog/pkg/config" "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" ) func prepareDockerfileDir(buildDir string, dockerfileContents string) (string, error) { @@ -35,6 +37,10 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( return buildkitclient.SolveOpt{}, err } + fmt.Printf("workingdir %q\ncontextdir %q\n", opts.WorkingDir, opts.ContextDir) + + // Secrets []string + // first, configure the frontend, in this case, dockerfile.v0 frontendAttrs := map[string]string{ // filename is the path to the Dockerfile within the "dockerfile" LocalDir context @@ -64,19 +70,45 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( frontendAttrs["build-arg:"+k] = *v } + // Add SOURCE_DATE_EPOCH if Epoch is set + if opts.Epoch != nil && *opts.Epoch >= 0 { + frontendAttrs["build-arg:SOURCE_DATE_EPOCH"] = fmt.Sprintf("%d", *opts.Epoch) + } + + localDirs := map[string]string{ + "dockerfile": filepath.Dir(dockerfilePath), + "context": opts.ContextDir, + } + + // Add user-supplied build contexts, but don't overwrite 'dockerfile' or 'context' + for name, dir := range opts.BuildContexts { + if name == "dockerfile" || name == "context" { + console.Warnf("build context name collision: %q", name) + continue + } + localDirs[name] = dir + } + + // Set exporter attributes + exporterAttrs := map[string]string{ + "name": opts.ImageName, + } + + // if SOURCE_DATE_EPOCH is present in the build args, tell the frontend to rewrite timestamps + if _, ok := frontendAttrs["build-arg:SOURCE_DATE_EPOCH"]; ok { + exporterAttrs["rewrite-timestamp"] = "true" + } + solveOpts := buildkitclient.SolveOpt{ Frontend: "dockerfile.v0", FrontendAttrs: frontendAttrs, - LocalDirs: map[string]string{ - "dockerfile": filepath.Dir(dockerfilePath), - "context": opts.WorkingDir, - }, + LocalDirs: localDirs, // Docker Engine's worker only supports three exporters. // "moby" exporter works best for cog, since we want to keep images in // Docker Engine's image store. The others are exporting images to somewhere else. // https://github.com/moby/moby/blob/v20.10.24/builder/builder-next/worker/worker.go#L221 Exports: []buildkitclient.ExportEntry{ - {Type: "moby", Attrs: map[string]string{"name": opts.ImageName}}, + {Type: "moby", Attrs: exporterAttrs}, }, } @@ -99,14 +131,29 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( ) } + // Set cache imports/exports to match DockerCommand logic + // If cogconfig.BuildXCachePath is set, use local cache; otherwise, use inline + if cogconfig.BuildXCachePath != "" { + solveOpts.CacheImports = []buildkitclient.CacheOptionsEntry{ + {Type: "local", Attrs: map[string]string{"src": cogconfig.BuildXCachePath}}, + } + solveOpts.CacheExports = []buildkitclient.CacheOptionsEntry{ + {Type: "local", Attrs: map[string]string{"dest": cogconfig.BuildXCachePath}}, + } + } else { + solveOpts.CacheExports = []buildkitclient.CacheOptionsEntry{ + {Type: "inline"}, + } + } + return solveOpts, nil } -func newDisplay(statusCh chan *buildkitclient.SolveStatus) func() error { +func newDisplay(statusCh chan *buildkitclient.SolveStatus, displayMode string) func() error { return func() error { display, err := progressui.NewDisplay( os.Stderr, - progressui.DisplayMode(os.Getenv("BUILDKIT_PROGRESS")), + progressui.DisplayMode(displayMode), // progressui.WithPhase("BUILDINGGGGG"), // progressui.WithDesc("SOMETEXT", "SOMECONSOLE"), ) @@ -121,7 +168,6 @@ func newDisplay(statusCh chan *buildkitclient.SolveStatus) func() error { // See https://github.com/superfly/flyctl/pull/2682 for the context. _, err = display.UpdateFrom(context.Background(), statusCh) return err - } } From b910ed421e08379c9d87c009d7c6660b9d265e02 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 14:13:30 -0600 Subject: [PATCH 23/43] fix hang when an error occurs building the solve graph --- pkg/docker/api_client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index bb4f2d3a1c..daecbb7645 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -299,9 +299,6 @@ func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOp // Build the image. eg, ctx := errgroup.WithContext(ctx) - // run the display in a goroutine - eg.Go(newDisplay(statusCh, displayMode)) - // run the build in a goroutine eg.Go(func() error { options, err := solveOptFromImageOptions(buildDir, options) @@ -309,6 +306,9 @@ func (c *apiClient) ImageBuild(ctx context.Context, options command.ImageBuildOp return err } + // run the display in a goroutine _after_ we've built SolveOpt + eg.Go(newDisplay(statusCh, displayMode)) + res, err = bc.Solve(ctx, nil, options, statusCh) if err != nil { return err From 95e6eb618527fb7e3ab28293baed031df0df7fd3 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 14:14:38 -0600 Subject: [PATCH 24/43] buildx secrets, integration test for secrets --- pkg/docker/build_secrets.go | 79 +++++++++++++++++++ pkg/docker/buildkit.go | 15 ++-- pkg/docker/command/command.go | 1 - .../fixtures/secrets-project/cog.yaml | 24 ++++++ .../fixtures/secrets-project/file-secret.txt | 1 + .../fixtures/secrets-project/predict.py | 6 ++ .../test_integration/test_build.py | 21 +++++ 7 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 pkg/docker/build_secrets.go create mode 100644 test-integration/test_integration/fixtures/secrets-project/cog.yaml create mode 100644 test-integration/test_integration/fixtures/secrets-project/file-secret.txt create mode 100644 test-integration/test_integration/fixtures/secrets-project/predict.py diff --git a/pkg/docker/build_secrets.go b/pkg/docker/build_secrets.go new file mode 100644 index 0000000000..b05c3b03dd --- /dev/null +++ b/pkg/docker/build_secrets.go @@ -0,0 +1,79 @@ +package docker + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/moby/buildkit/session/secrets" + "github.com/moby/buildkit/session/secrets/secretsprovider" + "github.com/pkg/errors" + "github.com/tonistiigi/go-csvvalue" +) + +func ParseSecretsFromHost(workingDir string, secrets []string) (secrets.SecretStore, error) { + sources := make([]secretsprovider.Source, 0, len(secrets)) + + for _, secret := range secrets { + src, err := parseSecretFromHost(workingDir, secret) + if err != nil { + return nil, err + } + fmt.Println("src", src) + sources = append(sources, *src) + } + + fmt.Println("sources", sources) + store, err := secretsprovider.NewStore(sources) + if err != nil { + return nil, err + } + fmt.Println("store", store) + + return store, nil +} + +func parseSecretFromHost(workingDir, secret string) (*secretsprovider.Source, error) { + fields, err := csvvalue.Fields(secret, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse csv secret: %w", err) + } + + src := secretsprovider.Source{} + + var typ string + for _, field := range fields { + key, value, ok := strings.Cut(field, "=") + if !ok { + return nil, errors.Errorf("invalid field %q must be a key=value pair", field) + } + key = strings.ToLower(key) + switch key { + case "type": + if value != "file" && value != "env" { + return nil, errors.Errorf("unsupported secret type %q", value) + } + typ = value + case "id": + src.ID = value + case "source", "src": + if !filepath.IsAbs(value) { + value = filepath.Join(workingDir, value) + value, err = filepath.Abs(value) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for %q: %w", value, err) + } + } + src.FilePath = value + case "env": + src.Env = value + default: + return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field) + } + } + if typ == "env" && src.Env == "" { + src.Env = src.FilePath + src.FilePath = "" + } + return &src, nil +} diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go index 87a1210fdb..577c509e4b 100644 --- a/pkg/docker/buildkit.go +++ b/pkg/docker/buildkit.go @@ -119,16 +119,13 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( ) // add secrets to the session - if len(opts.BuildSecrets) > 0 { - secrets := make(map[string][]byte) - for k, v := range opts.BuildSecrets { - secrets[k] = []byte(v) + if len(opts.Secrets) > 0 { + // TODO[md]: support secrets direct from input in addition to env+file + store, err := ParseSecretsFromHost(opts.WorkingDir, opts.Secrets) + if err != nil { + return buildkitclient.SolveOpt{}, fmt.Errorf("failed to parse secrets: %w", err) } - - solveOpts.Session = append( - solveOpts.Session, - secretsprovider.FromMap(secrets), - ) + solveOpts.Session = append(solveOpts.Session, secretsprovider.NewSecretProvider(store)) } // Set cache imports/exports to match DockerCommand logic diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index af28faa6ac..029dd3b22b 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -40,7 +40,6 @@ type ImageBuildOptions struct { Labels map[string]string // only supported on buildkit client, not cli client - BuildSecrets map[string]string BuildArgs map[string]*string } diff --git a/test-integration/test_integration/fixtures/secrets-project/cog.yaml b/test-integration/test_integration/fixtures/secrets-project/cog.yaml new file mode 100644 index 0000000000..01d1b26606 --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/cog.yaml @@ -0,0 +1,24 @@ +build: + run: + # assert that the file secret of file-secret.txt on the host is written to the target file and has the expected value + - command: >- + ID="file-secret"; + EXPECTED_VALUE="file_secret_value"; + EXPECTED_PATH="/etc/file_secret.txt"; + [ "$(cat "$EXPECTED_PATH")" = "$EXPECTED_VALUE" ] || ( echo "Assertion failed \"$EXPECTED_PATH\" \"$(cat $EXPECTED_PATH)\" != \"$EXPECTED_VALUE\""; exit 1; ) + mounts: + - type: secret + id: file-secret + target: /etc/file_secret.txt + # assert that the env secret of $ENV_SECRET on the host is written to the target file and has the expected value + - command: >- + ID="env-secret"; + EXPECTED_VALUE="env_secret_value"; + EXPECTED_PATH="/var/env-secret.txt"; + [ "$(cat "$EXPECTED_PATH")" = "$EXPECTED_VALUE" ] || ( echo "Assertion failed \"$EXPECTED_PATH\" \"$(cat $EXPECTED_PATH)\" != \"$EXPECTED_VALUE\""; exit 1; ) + mounts: + - type: secret + id: env-secret + target: /var/env-secret.txt + +predict: "predict.py:Predictor" diff --git a/test-integration/test_integration/fixtures/secrets-project/file-secret.txt b/test-integration/test_integration/fixtures/secrets-project/file-secret.txt new file mode 100644 index 0000000000..dd7af47dbc --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/file-secret.txt @@ -0,0 +1 @@ +file_secret_value diff --git a/test-integration/test_integration/fixtures/secrets-project/predict.py b/test-integration/test_integration/fixtures/secrets-project/predict.py new file mode 100644 index 0000000000..95a24e7178 --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/predict.py @@ -0,0 +1,6 @@ +from cog import BasePredictor + + +class Predictor(BasePredictor): + def predict(self, num: int) -> int: + return num * 2 diff --git a/test-integration/test_integration/test_build.py b/test-integration/test_integration/test_build.py index 30ce90d201..8c1ac12750 100644 --- a/test-integration/test_integration/test_build.py +++ b/test-integration/test_integration/test_build.py @@ -534,3 +534,24 @@ def test_install_requires_packaging(docker_image, cog_binary): ) print(build_process.stderr.decode()) assert build_process.returncode == 0 + + +def test_secrets(tmpdir_factory, docker_image, cog_binary): + project_dir = Path(__file__).parent / "fixtures/secrets-project" + + build_process = subprocess.run( + [ + cog_binary, + "build", + "-t", + docker_image, + "--secret", + "id=file-secret,src=file-secret.txt", + "--secret", + "id=env-secret,env=ENV_SECRET", + ], + cwd=project_dir, + capture_output=True, + env={**os.environ, "ENV_SECRET": "env_secret_value"}, + ) + assert build_process.returncode == 0 From a0fe3d1acc2aaa640da0f1322cde94bbc6b740bc Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 14:14:49 -0600 Subject: [PATCH 25/43] mod tidy --- go.mod | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 35559838fd..81cce05d2c 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/moby/buildkit v0.21.1 github.com/moby/term v0.5.2 + github.com/opencontainers/image-spec v1.1.1 + github.com/pkg/errors v0.9.1 github.com/replicate/go v0.0.0-20250205165008-b772d7cd506b github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/spf13/cobra v1.9.1 @@ -26,6 +28,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.37.0 github.com/testcontainers/testcontainers-go/modules/registry v0.37.0 + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 github.com/vbauerster/mpb/v8 v8.9.1 github.com/vincent-petithory/dataurl v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -35,6 +38,7 @@ require ( golang.org/x/sync v0.13.0 golang.org/x/sys v0.33.0 golang.org/x/term v0.31.0 + google.golang.org/grpc v1.71.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/yaml v1.4.0 @@ -214,10 +218,8 @@ require ( github.com/nunnatsa/ginkgolinter v0.19.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.7.1 // indirect @@ -267,7 +269,6 @@ require ( github.com/tomarrell/wrapcheck/v2 v2.10.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583 // indirect - github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect github.com/ultraware/funlen v0.2.0 // indirect @@ -306,7 +307,6 @@ require ( golang.org/x/tools v0.32.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 // indirect - google.golang.org/grpc v1.71.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gotest.tools/gotestsum v1.12.2 // indirect honnef.co/go/tools v0.6.1 // indirect From 9ea54399d58c74507a8d8433ce7479386e102ffa Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 14:16:33 -0600 Subject: [PATCH 26/43] lint fixes --- pkg/docker/buildkit.go | 1 + pkg/docker/command/command.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go index 577c509e4b..38f6082b11 100644 --- a/pkg/docker/buildkit.go +++ b/pkg/docker/buildkit.go @@ -45,6 +45,7 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( frontendAttrs := map[string]string{ // filename is the path to the Dockerfile within the "dockerfile" LocalDir context "filename": filepath.Base(dockerfilePath), + // TODO[md]: support multi-stage target // target is the name of a stage in a multi-stage Dockerfile // "target": opts.Target, // Replicate only supports linux/amd64, but local Docker Engine could be running on ARM, diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index 029dd3b22b..6be650c0ee 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -30,7 +30,9 @@ type ImageBuildOptions struct { WorkingDir string DockerfileContents string // TODO[md]: ImageName should be renamed to Tag - ImageName string + ImageName string + // Secrets in the format of "id=foo,src=/path/to/file" or "id=kube,env=KUBECONFIG" + // docs: https://docs.docker.com/build/building/secrets/#use-secrets-in-dockerfile Secrets []string NoCache bool ProgressOutput string @@ -40,7 +42,7 @@ type ImageBuildOptions struct { Labels map[string]string // only supported on buildkit client, not cli client - BuildArgs map[string]*string + BuildArgs map[string]*string } type RunOptions struct { From a7f01e9643dd9d54da67b3db8f96e389110f836c Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 14:19:00 -0600 Subject: [PATCH 27/43] format fix --- pkg/docker/docker_command.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index b6961d8448..d81e0f4645 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -196,7 +196,6 @@ func (c *DockerCommand) ContainerStop(ctx context.Context, containerID string) e containerID, } - if err := c.exec(ctx, nil, io.Discard, nil, "", args); err != nil { if isContainerNotFoundError(err) { err = &command.NotFoundError{Object: "container", Ref: containerID} From abcd3170aa02cfc27378aed169e56542c0eedfaf Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 15:01:33 -0600 Subject: [PATCH 28/43] fix race in mock docker registry port --- pkg/docker/docker_client_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index 048ecb731f..d7360b9557 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -3,7 +3,6 @@ package docker import ( "bytes" "net" - "strconv" "strings" "testing" @@ -18,7 +17,6 @@ import ( "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/docker/dockertest" "github.com/replicate/cog/pkg/registry_testhelpers" - "github.com/replicate/cog/pkg/util" ) func TestDockerClient(t *testing.T) { @@ -319,10 +317,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { t.Parallel() // start a local tcp server that immediately closes connections - port, err := util.PickFreePort(2000, 9999) - require.NoError(t, err, "Failed to pick free tcp port") - addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) - listener, err := net.Listen("tcp", addr) + listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer listener.Close() @@ -337,7 +332,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { }() // Create a reference to the mock registry - ref := dockertest.NewRef(t).WithRegistry(addr) + ref := dockertest.NewRef(t).WithRegistry(listener.Addr().String()) dockerHelper.ImageFixture(t, "alpine", ref.String()) // Try to push to the mock registry From be3c83b9f283c45e1996b6c5ff619857566049fb Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 15:28:25 -0600 Subject: [PATCH 29/43] integration test for secrets --- .../fixtures/secrets-project/cog.yaml | 24 +++++++++++++++++++ .../fixtures/secrets-project/file-secret.txt | 1 + .../fixtures/secrets-project/predict.py | 6 +++++ .../test_integration/test_build.py | 21 ++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 test-integration/test_integration/fixtures/secrets-project/cog.yaml create mode 100644 test-integration/test_integration/fixtures/secrets-project/file-secret.txt create mode 100644 test-integration/test_integration/fixtures/secrets-project/predict.py diff --git a/test-integration/test_integration/fixtures/secrets-project/cog.yaml b/test-integration/test_integration/fixtures/secrets-project/cog.yaml new file mode 100644 index 0000000000..01d1b26606 --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/cog.yaml @@ -0,0 +1,24 @@ +build: + run: + # assert that the file secret of file-secret.txt on the host is written to the target file and has the expected value + - command: >- + ID="file-secret"; + EXPECTED_VALUE="file_secret_value"; + EXPECTED_PATH="/etc/file_secret.txt"; + [ "$(cat "$EXPECTED_PATH")" = "$EXPECTED_VALUE" ] || ( echo "Assertion failed \"$EXPECTED_PATH\" \"$(cat $EXPECTED_PATH)\" != \"$EXPECTED_VALUE\""; exit 1; ) + mounts: + - type: secret + id: file-secret + target: /etc/file_secret.txt + # assert that the env secret of $ENV_SECRET on the host is written to the target file and has the expected value + - command: >- + ID="env-secret"; + EXPECTED_VALUE="env_secret_value"; + EXPECTED_PATH="/var/env-secret.txt"; + [ "$(cat "$EXPECTED_PATH")" = "$EXPECTED_VALUE" ] || ( echo "Assertion failed \"$EXPECTED_PATH\" \"$(cat $EXPECTED_PATH)\" != \"$EXPECTED_VALUE\""; exit 1; ) + mounts: + - type: secret + id: env-secret + target: /var/env-secret.txt + +predict: "predict.py:Predictor" diff --git a/test-integration/test_integration/fixtures/secrets-project/file-secret.txt b/test-integration/test_integration/fixtures/secrets-project/file-secret.txt new file mode 100644 index 0000000000..dd7af47dbc --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/file-secret.txt @@ -0,0 +1 @@ +file_secret_value diff --git a/test-integration/test_integration/fixtures/secrets-project/predict.py b/test-integration/test_integration/fixtures/secrets-project/predict.py new file mode 100644 index 0000000000..95a24e7178 --- /dev/null +++ b/test-integration/test_integration/fixtures/secrets-project/predict.py @@ -0,0 +1,6 @@ +from cog import BasePredictor + + +class Predictor(BasePredictor): + def predict(self, num: int) -> int: + return num * 2 diff --git a/test-integration/test_integration/test_build.py b/test-integration/test_integration/test_build.py index 30ce90d201..8c1ac12750 100644 --- a/test-integration/test_integration/test_build.py +++ b/test-integration/test_integration/test_build.py @@ -534,3 +534,24 @@ def test_install_requires_packaging(docker_image, cog_binary): ) print(build_process.stderr.decode()) assert build_process.returncode == 0 + + +def test_secrets(tmpdir_factory, docker_image, cog_binary): + project_dir = Path(__file__).parent / "fixtures/secrets-project" + + build_process = subprocess.run( + [ + cog_binary, + "build", + "-t", + docker_image, + "--secret", + "id=file-secret,src=file-secret.txt", + "--secret", + "id=env-secret,env=ENV_SECRET", + ], + cwd=project_dir, + capture_output=True, + env={**os.environ, "ENV_SECRET": "env_secret_value"}, + ) + assert build_process.returncode == 0 From 6be52c4bfaec5cbe3f27937fe8195a5192982727 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:26:34 -0600 Subject: [PATCH 30/43] move credentials logic off command.Command the credential stuff isn't tied to the docker client. This moves logic to helper functions that both clients can use --- pkg/docker/credentials.go | 150 +++++++++++++++++++++++++++++++++++ pkg/docker/docker_command.go | 69 +--------------- pkg/docker/login.go | 2 +- 3 files changed, 153 insertions(+), 68 deletions(-) create mode 100644 pkg/docker/credentials.go diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go new file mode 100644 index 0000000000..722dc99851 --- /dev/null +++ b/pkg/docker/credentials.go @@ -0,0 +1,150 @@ +package docker + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/config/types" + "github.com/docker/docker/api/types/registry" + + "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" +) + +func loadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { + conf := config.LoadDefaultConfigFile(os.Stderr) + credsStore := conf.CredentialsStore + if credsStore == "" { + authConf, err := loadAuthFromConfig(conf, registryHost) + if err != nil { + return nil, err + } + return &command.UserInfo{ + Token: authConf.Password, + Username: authConf.Username, + }, nil + } + credsHelper, err := loadAuthFromCredentialsStore(ctx, credsStore, registryHost) + if err != nil { + return nil, err + } + return &command.UserInfo{ + Token: credsHelper.Secret, + Username: credsHelper.Username, + }, nil +} + +func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types.AuthConfig, error) { + return conf.AuthConfigs[registryHost], nil +} + +func loadRegistryAuths(ctx context.Context, registryHosts ...string) (map[string]registry.AuthConfig, error) { + conf := config.LoadDefaultConfigFile(os.Stderr) + + out := make(map[string]registry.AuthConfig) + + for _, host := range registryHosts { + console.Debugf("=== loadRegistryAuths %s", host) + // check the credentials store first if set + if conf.CredentialsStore != "" { + console.Debugf("=== loadRegistryAuths %s: credentials store set", host) + credsHelper, err := loadAuthFromCredentialsStore(ctx, conf.CredentialsStore, host) + if err != nil { + console.Debugf("=== loadRegistryAuths %s: error loading credentials store: %s", host, err) + return nil, err + } + console.Debugf("=== loadRegistryAuths %s: credentials store loaded", host) + out[host] = registry.AuthConfig{ + Username: credsHelper.Username, + Password: credsHelper.Secret, + ServerAddress: host, + } + continue + } + + // next, check if the auth config exists in the config file + if auth, ok := conf.AuthConfigs[host]; ok { + console.Debugf("=== loadRegistryAuths %s: auth config found in config file", host) + out[host] = registry.AuthConfig{ + Username: auth.Username, + Password: auth.Password, + Auth: auth.Auth, + Email: auth.Email, + ServerAddress: host, + IdentityToken: auth.IdentityToken, + RegistryToken: auth.RegistryToken, + } + continue + } + + console.Debugf("=== loadRegistryAuths %s: no auth config found", host) + + // TODO[md]: should we error here!? probably not... + } + + return out, nil +} + +// func loadRegistryAuths(conf *configfile.ConfigFile, registryHosts ...string) (map[string]types.AuthConfig, error) { +// out := make(map[string]types.AuthConfig) +// for _, host := range registryHosts { +// auth, ok := conf.AuthConfigs[host] +// if !ok { +// // TODO[md]: should we return an error here or just carry on? +// return nil, fmt.Errorf("no auth config found for registry host %s", host) +// } +// out[host] = auth +// } + +// return out, nil +// } + +func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { + var out strings.Builder + binary := dockerCredentialBinary(credsStore) + cmd := exec.CommandContext(ctx, binary, "get") + cmd.Env = os.Environ() + cmd.Stdout = &out + cmd.Stderr = &out + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + defer stdin.Close() + console.Debug("$ " + strings.Join(cmd.Args, " ")) + err = cmd.Start() + if err != nil { + return nil, err + } + _, err = io.WriteString(stdin, registryHost) + if err != nil { + return nil, err + } + err = stdin.Close() + if err != nil { + return nil, err + } + err = cmd.Wait() + if err != nil { + return nil, fmt.Errorf("exec wait error: %w", err) + } + + var config CredentialHelperInput + err = json.Unmarshal([]byte(out.String()), &config) + if err != nil { + return nil, err + } + + return &config, nil +} + +func dockerCredentialBinary(credsStore string) string { + return "docker-credential-" + credsStore +} diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 621f78f23c..09be542c5d 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -80,29 +80,10 @@ func (c *DockerCommand) Push(ctx context.Context, image string) error { return c.exec(ctx, nil, nil, nil, "", []string{"push", image}) } +// TODO[md]: this doesn't need to be on the interface, move to auth handler func (c *DockerCommand) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== DockerCommand.LoadUserInformation %s", registryHost) - conf := config.LoadDefaultConfigFile(os.Stderr) - credsStore := conf.CredentialsStore - if credsStore == "" { - authConf, err := loadAuthFromConfig(conf, registryHost) - if err != nil { - return nil, err - } - return &command.UserInfo{ - Token: authConf.Password, - Username: authConf.Username, - }, nil - } - credsHelper, err := loadAuthFromCredentialsStore(ctx, credsStore, registryHost) - if err != nil { - return nil, err - } - return &command.UserInfo{ - Token: credsHelper.Secret, - Username: credsHelper.Username, - }, nil } func (c *DockerCommand) CreateTarFile(ctx context.Context, image string, tmpDir string, tarFile string, folder string) (string, error) { @@ -151,6 +132,7 @@ func (c *DockerCommand) CreateAptTarFile(ctx context.Context, tmpDir string, apt } return aptTarFile, nil + return loadUserInformation(ctx, registryHost) } func (c *DockerCommand) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) { @@ -503,50 +485,3 @@ func (c *DockerCommand) execCaptured(ctx context.Context, in io.Reader, dir stri } return out.String(), nil } - -func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types.AuthConfig, error) { - return conf.AuthConfigs[registryHost], nil -} - -func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { - var out strings.Builder - binary := DockerCredentialBinary(credsStore) - cmd := exec.CommandContext(ctx, binary, "get") - cmd.Env = os.Environ() - cmd.Stdout = &out - cmd.Stderr = &out - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - defer stdin.Close() - console.Debug("$ " + strings.Join(cmd.Args, " ")) - err = cmd.Start() - if err != nil { - return nil, err - } - _, err = io.WriteString(stdin, registryHost) - if err != nil { - return nil, err - } - err = stdin.Close() - if err != nil { - return nil, err - } - err = cmd.Wait() - if err != nil { - return nil, fmt.Errorf("exec wait error: %w", err) - } - - var config CredentialHelperInput - err = json.Unmarshal([]byte(out.String()), &config) - if err != nil { - return nil, err - } - - return &config, nil -} - -func DockerCredentialBinary(credsStore string) string { - return "docker-credential-" + credsStore -} diff --git a/pkg/docker/login.go b/pkg/docker/login.go index e17f2e123c..c9f5b18706 100644 --- a/pkg/docker/login.go +++ b/pkg/docker/login.go @@ -37,7 +37,7 @@ func saveAuthToConfig(conf *configfile.ConfigFile, registryHost string, username } func saveAuthToCredentialsStore(ctx context.Context, credsStore string, registryHost string, username string, token string) error { - binary := DockerCredentialBinary(credsStore) + binary := dockerCredentialBinary(credsStore) input := CredentialHelperInput{ Username: username, Secret: token, From 33559ae97b21226e1c47d0e47773d1091a9cb18f Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:31:30 -0600 Subject: [PATCH 31/43] move CreateTarFile & CreateAptTarFile to helpers The monobase helpers from command.Command were calling docker run, instead have them call the run functions on the client since the logic doesn't depend on client implementation --- pkg/docker/apt.go | 81 ----------- pkg/docker/command/command.go | 2 - pkg/docker/docker_command.go | 52 ------- pkg/docker/dockertest/mock_command.go | 10 ++ pkg/docker/fast_push.go | 4 +- pkg/docker/monobase.go | 141 +++++++++++++++++++ pkg/docker/{apt_test.go => monobase_test.go} | 0 7 files changed, 153 insertions(+), 137 deletions(-) delete mode 100644 pkg/docker/apt.go create mode 100644 pkg/docker/monobase.go rename pkg/docker/{apt_test.go => monobase_test.go} (100%) diff --git a/pkg/docker/apt.go b/pkg/docker/apt.go deleted file mode 100644 index 10a69643ff..0000000000 --- a/pkg/docker/apt.go +++ /dev/null @@ -1,81 +0,0 @@ -package docker - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/replicate/cog/pkg/docker/command" -) - -const aptTarballPrefix = "apt." -const aptTarballSuffix = ".tar.zst" - -func CreateAptTarball(ctx context.Context, tmpDir string, dockerCommand command.Command, packages ...string) (string, error) { - if len(packages) > 0 { - sort.Strings(packages) - hash := sha256.New() - hash.Write([]byte(strings.Join(packages, " "))) - hexHash := hex.EncodeToString(hash.Sum(nil)) - aptTarFile := aptTarballPrefix + hexHash + aptTarballSuffix - aptTarPath := filepath.Join(tmpDir, aptTarFile) - - if _, err := os.Stat(aptTarPath); errors.Is(err, os.ErrNotExist) { - // Remove previous apt tar files. - err = removeAptTarballs(tmpDir) - if err != nil { - return "", err - } - - // Create the apt tar file - _, err = dockerCommand.CreateAptTarFile(ctx, tmpDir, aptTarFile, packages...) - if err != nil { - return "", err - } - } - - return aptTarFile, nil - } - return "", nil -} - -func CurrentAptTarball(tmpDir string) (string, error) { - files, err := os.ReadDir(tmpDir) - if err != nil { - return "", fmt.Errorf("os read dir error: %w", err) - } - - for _, file := range files { - fileName := file.Name() - if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { - return filepath.Join(tmpDir, fileName), nil - } - } - - return "", nil -} - -func removeAptTarballs(tmpDir string) error { - files, err := os.ReadDir(tmpDir) - if err != nil { - return err - } - - for _, file := range files { - fileName := file.Name() - if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { - err = os.Remove(filepath.Join(tmpDir, fileName)) - if err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/docker/command/command.go b/pkg/docker/command/command.go index c2a67a86a6..f397fc7485 100644 --- a/pkg/docker/command/command.go +++ b/pkg/docker/command/command.go @@ -15,8 +15,6 @@ type Command interface { Pull(ctx context.Context, ref string, force bool) (*image.InspectResponse, error) Push(ctx context.Context, ref string) error LoadUserInformation(ctx context.Context, registryHost string) (*UserInfo, error) - CreateTarFile(ctx context.Context, ref string, tmpDir string, tarFile string, folder string) (string, error) - CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) Inspect(ctx context.Context, ref string) (*image.InspectResponse, error) ImageExists(ctx context.Context, ref string) (bool, error) ContainerLogs(ctx context.Context, containerID string, w io.Writer) error diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 09be542c5d..29095a8ad0 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -9,14 +9,10 @@ import ( "io" "os" "os/exec" - "path/filepath" "runtime" "strings" "github.com/creack/pty" - "github.com/docker/cli/cli/config" - "github.com/docker/cli/cli/config/configfile" - "github.com/docker/cli/cli/config/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/mattn/go-isatty" @@ -84,54 +80,6 @@ func (c *DockerCommand) Push(ctx context.Context, image string) error { func (c *DockerCommand) LoadUserInformation(ctx context.Context, registryHost string) (*command.UserInfo, error) { console.Debugf("=== DockerCommand.LoadUserInformation %s", registryHost) -} - -func (c *DockerCommand) CreateTarFile(ctx context.Context, image string, tmpDir string, tarFile string, folder string) (string, error) { - console.Debugf("=== DockerCommand.CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) - - args := []string{ - "run", - "--rm", - // force platform to linux/amd64 so darwin/arm64 outputs work in prod - "--platform", "linux/amd64", - "--volume", - tmpDir + ":/buildtmp", - image, - "/opt/r8/monobase/tar.sh", - "/buildtmp/" + tarFile, - "/", - folder, - } - if err := c.exec(ctx, nil, nil, nil, "", args); err != nil { - return "", err - } - return filepath.Join(tmpDir, tarFile), nil -} - -func (c *DockerCommand) CreateAptTarFile(ctx context.Context, tmpDir string, aptTarFile string, packages ...string) (string, error) { - console.Debugf("=== DockerCommand.CreateAptTarFile %s %s", aptTarFile, packages) - - // This uses a hardcoded monobase image to produce an apt tar file. - // The reason being that this apt tar file is created outside the docker file, and it is created by - // running the apt.sh script on the monobase with the packages we intend to install, which produces - // a tar file that can be untarred into a docker build to achieve the equivalent of an apt-get install. - args := []string{ - "run", - "--rm", - // force platform to linux/amd64 so darwin/arm64 outputs work in prod - "--platform", "linux/amd64", - "--volume", - tmpDir + ":/buildtmp", - "r8.im/monobase:latest", - "/opt/r8/monobase/apt.sh", - "/buildtmp/" + aptTarFile, - } - args = append(args, packages...) - if err := c.exec(ctx, nil, nil, nil, "", args); err != nil { - return "", err - } - - return aptTarFile, nil return loadUserInformation(ctx, registryHost) } diff --git a/pkg/docker/dockertest/mock_command.go b/pkg/docker/dockertest/mock_command.go index e27c715ed2..5b399d9aab 100644 --- a/pkg/docker/dockertest/mock_command.go +++ b/pkg/docker/dockertest/mock_command.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" @@ -99,6 +100,15 @@ func (c *MockCommand) ImageBuild(ctx context.Context, options command.ImageBuild } func (c *MockCommand) Run(ctx context.Context, options command.RunOptions) error { + // hack to handle generating tar files for monobase + if options.Args[0] == "/opt/r8/monobase/tar.sh" || options.Args[0] == "/opt/r8/monobase/apt.sh" { + tmpDir := options.Volumes[0].Source + tarfile := strings.TrimPrefix(options.Args[1], "/buildtmp/") + + outPath := filepath.Join(tmpDir, tarfile) + return os.WriteFile(outPath, []byte("hello\ngo\n"), 0o644) + } + panic("not implemented") } diff --git a/pkg/docker/fast_push.go b/pkg/docker/fast_push.go index f67bd92766..b002c57d27 100644 --- a/pkg/docker/fast_push.go +++ b/pkg/docker/fast_push.go @@ -147,11 +147,11 @@ func FastPush(ctx context.Context, image string, projectDir string, command comm } func createPythonPackagesTarFile(ctx context.Context, image string, tmpDir string, command command.Command) (string, error) { - return command.CreateTarFile(ctx, image, tmpDir, requirementsTarFile, "root/.venv") + return CreateTarFile(ctx, command, image, tmpDir, requirementsTarFile, "root/.venv") } func createSrcTarFile(ctx context.Context, image string, tmpDir string, command command.Command) (string, error) { - return command.CreateTarFile(ctx, image, tmpDir, "src.tar.zst", "src") + return CreateTarFile(ctx, command, image, tmpDir, "src.tar.zst", "src") } func createWeightsFilesFromWeightsManifest(weights []weights.Weight) []web.File { diff --git a/pkg/docker/monobase.go b/pkg/docker/monobase.go new file mode 100644 index 0000000000..f604a65359 --- /dev/null +++ b/pkg/docker/monobase.go @@ -0,0 +1,141 @@ +package docker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/replicate/cog/pkg/docker/command" + "github.com/replicate/cog/pkg/util/console" +) + +const aptTarballPrefix = "apt." +const aptTarballSuffix = ".tar.zst" + +func CreateAptTarball(ctx context.Context, tmpDir string, dockerClient command.Command, packages ...string) (string, error) { + if len(packages) > 0 { + sort.Strings(packages) + hash := sha256.New() + hash.Write([]byte(strings.Join(packages, " "))) + hexHash := hex.EncodeToString(hash.Sum(nil)) + aptTarFile := aptTarballPrefix + hexHash + aptTarballSuffix + aptTarPath := filepath.Join(tmpDir, aptTarFile) + + if _, err := os.Stat(aptTarPath); errors.Is(err, os.ErrNotExist) { + // Remove previous apt tar files. + err = removeAptTarballs(tmpDir) + if err != nil { + return "", err + } + + // Create the apt tar file + _, err = CreateAptTarFile(ctx, dockerClient, tmpDir, aptTarFile, packages...) + if err != nil { + return "", err + } + } + + return aptTarFile, nil + } + return "", nil +} + +func CurrentAptTarball(tmpDir string) (string, error) { + files, err := os.ReadDir(tmpDir) + if err != nil { + return "", fmt.Errorf("os read dir error: %w", err) + } + + for _, file := range files { + fileName := file.Name() + if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { + return filepath.Join(tmpDir, fileName), nil + } + } + + return "", nil +} + +func removeAptTarballs(tmpDir string) error { + files, err := os.ReadDir(tmpDir) + if err != nil { + return err + } + + for _, file := range files { + fileName := file.Name() + if strings.HasPrefix(fileName, aptTarballPrefix) && strings.HasSuffix(fileName, aptTarballSuffix) { + err = os.Remove(filepath.Join(tmpDir, fileName)) + if err != nil { + return err + } + } + } + + return nil +} + +func CreateTarFile(ctx context.Context, dockerClient command.Command, image string, tmpDir string, tarFile string, folder string) (string, error) { + console.Debugf("=== CreateTarFile %s %s %s %s", image, tmpDir, tarFile, folder) + + opts := command.RunOptions{ + Image: image, + Args: []string{ + "/opt/r8/monobase/tar.sh", + path.Join("/buildtmp", tarFile), + "/", + folder, + }, + Volumes: []command.Volume{ + { + Source: tmpDir, + Destination: "/buildtmp", + }, + }, + } + + if err := dockerClient.Run(ctx, opts); err != nil { + return "", err + } + + return filepath.Join(tmpDir, tarFile), nil +} + +func CreateAptTarFile(ctx context.Context, dockerClient command.Command, tmpDir string, aptTarFile string, packages ...string) (string, error) { + console.Debugf("=== CreateAptTarFile %s %s", aptTarFile, packages) + + // This uses a hardcoded monobase image to produce an apt tar file. + // The reason being that this apt tar file is created outside the docker file, and it is created by + // running the apt.sh script on the monobase with the packages we intend to install, which produces + // a tar file that can be untarred into a docker build to achieve the equivalent of an apt-get install. + + opts := command.RunOptions{ + Image: "r8.im/monobase:latest", + Args: append( + []string{ + "/opt/r8/monobase/apt.sh", + path.Join("/buildtmp", aptTarFile), + }, + packages..., + ), + Volumes: []command.Volume{ + { + Source: tmpDir, + Destination: "/buildtmp", + }, + }, + } + + if err := dockerClient.Run(ctx, opts); err != nil { + return "", err + } + + return aptTarFile, nil +} diff --git a/pkg/docker/apt_test.go b/pkg/docker/monobase_test.go similarity index 100% rename from pkg/docker/apt_test.go rename to pkg/docker/monobase_test.go From b8275a7e89fa23e636e575ca52ec37c149d7e30a Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:32:22 -0600 Subject: [PATCH 32/43] error helpers helpers to map errors from different backends to known types. needed to support both clients --- pkg/docker/command/errors.go | 2 ++ pkg/docker/docker_command.go | 36 +++++++++++++++++++++----- pkg/docker/errors.go | 49 ++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 pkg/docker/errors.go diff --git a/pkg/docker/command/errors.go b/pkg/docker/command/errors.go index 8bcc4ba9eb..66f35a1098 100644 --- a/pkg/docker/command/errors.go +++ b/pkg/docker/command/errors.go @@ -29,3 +29,5 @@ func (e *NotFoundError) Is(target error) bool { func IsNotFoundError(err error) bool { return errors.Is(err, &NotFoundError{}) } + +var ErrAuthorizationFailed = errors.New("authorization failed") diff --git a/pkg/docker/docker_command.go b/pkg/docker/docker_command.go index 29095a8ad0..d81e0f4645 100644 --- a/pkg/docker/docker_command.go +++ b/pkg/docker/docker_command.go @@ -73,7 +73,18 @@ func (c *DockerCommand) Pull(ctx context.Context, image string, force bool) (*im func (c *DockerCommand) Push(ctx context.Context, image string) error { console.Debugf("=== DockerCommand.Push %s", image) - return c.exec(ctx, nil, nil, nil, "", []string{"push", image}) + err := c.exec(ctx, nil, nil, nil, "", []string{"push", image}) + if err != nil { + if isTagNotFoundError(err) { + return &command.NotFoundError{Ref: image, Object: "tag"} + } + if isAuthorizationFailedError(err) { + return command.ErrAuthorizationFailed + } + return err + } + + return nil } // TODO[md]: this doesn't need to be on the interface, move to auth handler @@ -92,7 +103,7 @@ func (c *DockerCommand) Inspect(ctx context.Context, ref string) (*image.Inspect } output, err := c.execCaptured(ctx, nil, "", args) if err != nil { - if strings.Contains(err.Error(), "No such image") { + if isImageNotFoundError(err) { return nil, &command.NotFoundError{Object: "image", Ref: ref} } return nil, err @@ -137,7 +148,14 @@ func (c *DockerCommand) ContainerLogs(ctx context.Context, containerID string, w "--follow", } - return c.exec(ctx, nil, w, nil, "", args) + err := c.exec(ctx, nil, w, nil, "", args) + if err != nil { + if isContainerNotFoundError(err) { + return &command.NotFoundError{Ref: containerID, Object: "container"} + } + return err + } + return err } func (c *DockerCommand) ContainerInspect(ctx context.Context, id string) (*container.InspectResponse, error) { @@ -151,7 +169,7 @@ func (c *DockerCommand) ContainerInspect(ctx context.Context, id string) (*conta output, err := c.execCaptured(ctx, nil, "", args) if err != nil { - if strings.Contains(err.Error(), "No such container") { + if isContainerNotFoundError(err) { return nil, &command.NotFoundError{Object: "container", Ref: id} } return nil, err @@ -179,7 +197,7 @@ func (c *DockerCommand) ContainerStop(ctx context.Context, containerID string) e } if err := c.exec(ctx, nil, io.Discard, nil, "", args); err != nil { - if strings.Contains(err.Error(), "No such container") { + if isContainerNotFoundError(err) { err = &command.NotFoundError{Object: "container", Ref: containerID} } return fmt.Errorf("failed to stop container %q: %w", containerID, err) @@ -347,7 +365,7 @@ func (c *DockerCommand) containerRun(ctx context.Context, options command.RunOpt err := c.exec(ctx, options.Stdin, options.Stdout, options.Stderr, "", args) if err != nil { - if strings.Contains(err.Error(), "could not select device driver") || strings.Contains(err.Error(), "nvidia-container-cli: initialization error") { + if isMissingDeviceDriverError(err) { return ErrMissingDeviceDriver } return err @@ -420,6 +438,12 @@ func (c *DockerCommand) exec(ctx context.Context, in io.Reader, outw, errw io.Wr if errors.Is(err, context.Canceled) { return err } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + if !exitErr.Exited() && strings.Contains(exitErr.Error(), "signal: killed") { + return context.DeadlineExceeded + } + } return fmt.Errorf("command failed: %s: %w", stderrBuf.String(), err) } return nil diff --git a/pkg/docker/errors.go b/pkg/docker/errors.go new file mode 100644 index 0000000000..feac88a465 --- /dev/null +++ b/pkg/docker/errors.go @@ -0,0 +1,49 @@ +package docker + +import "strings" + +// Error messages vary between different backends (dockerd, containerd, podman, orbstack, etc) or even versions of docker. +// These helpers normalize the check so callers can handle situations without worrying about the underlying implementation. +// Yes, it's gross, but whattaya gonna do + +func isTagNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "tag does not exist") || + strings.Contains(msg, "An image does not exist locally with the tag") +} + +func isImageNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "image does not exist") || + strings.Contains(msg, "No such image") +} + +func isContainerNotFoundError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "container does not exist") || + strings.Contains(msg, "No such container") +} + +func isAuthorizationFailedError(err error) bool { + msg := err.Error() + + // registry requires auth and none were provided + if strings.Contains(msg, "no basic auth credentials") { + return true + } + + // registry rejected the provided auth + if strings.Contains(msg, "authorization failed") || + strings.Contains(msg, "401 Unauthorized") || + strings.Contains(msg, "unauthorized: authentication required") { + return true + } + + return false +} + +func isMissingDeviceDriverError(err error) bool { + msg := err.Error() + return strings.Contains(msg, "could not select device driver") || + strings.Contains(msg, "nvidia-container-cli: initialization error") +} From f2dfae46b8f53f6239437ba3d237122a594d24f2 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:33:46 -0600 Subject: [PATCH 33/43] more docker integration tests --- pkg/docker/docker_client_test.go | 499 +++++++++++++----- pkg/docker/dockertest/helper_client.go | 79 ++- pkg/docker/dockertest/ref.go | 82 +++ pkg/docker/dockertest/ref_test.go | 24 + .../registry_container.go | 98 +++- 5 files changed, 636 insertions(+), 146 deletions(-) create mode 100644 pkg/docker/dockertest/ref.go create mode 100644 pkg/docker/dockertest/ref_test.go diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index d5abbc919a..e3bcb93c51 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -1,12 +1,18 @@ package docker import ( + "bytes" + "net" + "strings" "testing" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/docker/dockertest" @@ -18,163 +24,414 @@ func TestDockerClient(t *testing.T) { t.Skip("skipping docker client tests in short mode") } - suite := &DockerClientSuite{ - dockerHelper: dockertest.NewHelperClient(t), - dockerClient: NewDockerCommand(), + client := NewDockerCommand() + runDockerClientTests(t, client) +} } - t.Run("ImageInspect", suite.runImageInspectTests) - t.Run("Pull", suite.runPullTests) - t.Run("ContainerStop", suite.runContainerStopTests) } -type DockerClientSuite struct { - dockerHelper *dockertest.HelperClient - dockerClient command.Command -} +func runDockerClientTests(t *testing.T, dockerClient command.Command) { + dockerHelper := dockertest.NewHelperClient(t) + testRegistry := registry_testhelpers.StartTestRegistry(t) -func (s *DockerClientSuite) assertImageExists(t *testing.T, imageRef string) { - inspect, err := s.dockerClient.Inspect(t.Context(), imageRef) - assert.NoError(t, err, "Failed to inspect image %q", imageRef) - assert.NotNil(t, inspect, "Image should exist") -} + dockerHelper.CleanupImages(t) -func (s *DockerClientSuite) assertNoImageExists(t *testing.T, imageRef string) { - inspect, err := s.dockerClient.Inspect(t.Context(), imageRef) - assert.ErrorIs(t, err, &command.NotFoundError{}, "Image should not exist") - assert.Nil(t, inspect, "Image should not exist") -} - -func (s *DockerClientSuite) runImageInspectTests(t *testing.T) { - t.Run("ExistingLocalImage", func(t *testing.T) { + t.Run("ImageInspect", func(t *testing.T) { t.Parallel() - image := "docker.io/library/busybox:latest" + t.Run("ExistingLocalImage", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + expectedImage := dockerHelper.InspectImage(t, ref.String()) + resp, err := dockerClient.Inspect(t.Context(), ref.String()) + require.NoError(t, err, "Failed to inspect image %q", ref.String()) + assert.Equal(t, expectedImage.ID, resp.ID) + }) - s.dockerHelper.MustPullImage(t, image) + t.Run("MissingLocalImage", func(t *testing.T) { + t.Parallel() - expectedImage := s.dockerHelper.InspectImage(t, image) - resp, err := s.dockerClient.Inspect(t.Context(), image) - require.NoError(t, err, "Failed to inspect image %q", image) - assert.Equal(t, expectedImage.ID, resp.ID) + image := "not-a-valid-image" + _, err := dockerClient.Inspect(t.Context(), image) + assert.ErrorIs(t, err, &command.NotFoundError{}) + assert.ErrorContains(t, err, "image not found") + }) }) - t.Run("MissingLocalImage", func(t *testing.T) { + t.Run("Pull", func(t *testing.T) { t.Parallel() - image := "not-a-valid-image" - _, err := s.dockerClient.Inspect(t.Context(), image) - assert.ErrorIs(t, err, &command.NotFoundError{}) - assert.ErrorContains(t, err, "image not found") + // TODO[md]: add tests for the following permutations: + // - remote reference exists/not exists + // - local reference exists/not exists + // - force pull true/false + + t.Run("RemoteImageExists", func(t *testing.T) { + t.Parallel() + repo := testRegistry.CloneRepoForTest(t, "alpine") + imageRef := repo + ":latest" + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + require.NoError(t, err, "Failed to pull image %q", imageRef) + dockerHelper.CleanupImage(t, imageRef) + + assertImageExists(t, dockerClient, imageRef) + expectedResp := dockerHelper.InspectImage(t, imageRef) + // TODO[md]: we should check that the responsees are actually equal beyond the IDs. but atm + // the CLI and api are slightly different. The CLI leaves the descriptor field nil while the + // API response is populated. These should be identical on the new client, so we can change to EqualValues + assert.Equal(t, expectedResp.ID, resp.ID, "inspect response should match expected") + }) + + t.Run("RemoteReferenceNotFound", func(t *testing.T) { + t.Parallel() + imageRef := testRegistry.ImageRefForTest(t, "") + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry + // so we handle other failure cases, like failed auth, unknown tag, and unknown repo + require.Error(t, err, "Failed to pull image %q", imageRef) + assert.ErrorIs(t, err, &command.NotFoundError{Object: "manifest", Ref: imageRef}) + assert.Nil(t, resp, "inspect response should be nil") + }) + + t.Run("InvalidAuth", func(t *testing.T) { + t.Skip("skip auth tests until we're using the docker engine since we can't set auth on the host without side effects") + imageRef := testRegistry.ImageRefForTest(t, "") + + assertNoImageExists(t, dockerClient, imageRef) + + resp, err := dockerClient.Pull(t.Context(), imageRef, false) + // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry + // so we handle other failure cases, like failed auth, unknown tag, and unknown repo + require.Error(t, err, "Failed to pull image %q", imageRef) + assert.ErrorContains(t, err, "failed to resolve reference") + assert.Nil(t, resp, "inspect response should be nil") + }) }) -} - -func (s *DockerClientSuite) runPullTests(t *testing.T) { - testRegistry := registry_testhelpers.StartTestRegistry(t) - // TODO[md]: add tests for the following permutations: - // - remote reference exists/not exists - // - local reference exists/not exists - // - force pull true/false + t.Run("ContainerStop", func(t *testing.T) { + t.Parallel() - t.Run("RemoteImageExists", func(t *testing.T) { - imageRef := testRegistry.ImageRefForTest(t, "") + t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sleep", "5000"), + ) + defer dockerHelper.CleanupImages(t) + defer testcontainers.CleanupContainer(t, container) + require.NoError(t, err, "Failed to run container") + + err = dockerClient.ContainerStop(t.Context(), container.ID) + require.NoError(t, err, "Failed to stop container %q", container.ID) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + }) + + t.Run("ContainerExistsAndIsNotRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.GenericContainer(t.Context(), + testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: testRegistry.ImageRef("alpine:latest"), + Cmd: []string{"sleep", "5000"}, + }, + Started: false, + }, + ) + defer testcontainers.CleanupContainer(t, container) + containerID := container.GetContainerID() + require.NoError(t, err, "Failed to create container") + + err = dockerClient.ContainerStop(t.Context(), containerID) + require.NoError(t, err, "Failed to stop container %q", containerID) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + }) + + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() + + err := dockerClient.ContainerStop(t.Context(), "containerid-that-does-not-exist") + require.ErrorIs(t, err, &command.NotFoundError{}) + require.ErrorContains(t, err, "container not found") + }) + }) - s.dockerHelper.LoadImageFixture(t, "alpine", imageRef) - s.dockerHelper.MustPushImage(t, imageRef) - s.dockerHelper.MustDeleteImage(t, imageRef) + t.Run("ContainerInspect", func(t *testing.T) { + t.Parallel() - s.assertNoImageExists(t, imageRef) + t.Run("ContainerExists", func(t *testing.T) { + t.Parallel() - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - require.NoError(t, err, "Failed to pull image %q", imageRef) - s.dockerHelper.CleanupImage(t, imageRef) + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sleep", "5000"), + ) + defer testcontainers.CleanupContainer(t, container) + require.NoError(t, err, "Failed to run container") - s.assertImageExists(t, imageRef) - expectedResp := s.dockerHelper.InspectImage(t, imageRef) - // TODO[md]: we should check that the responsees are actually equal beyond the IDs. but atm - // the CLI and api are slightly different. The CLI leaves the descriptor field nil while the - // API response is populated. These should be identical on the new client, so we can change to EqualValues - assert.Equal(t, expectedResp.ID, resp.ID, "inspect response should match expected") - }) + expected, err := container.Inspect(t.Context()) + require.NoError(t, err, "Failed to inspect container for expected response") - t.Run("RemoteReferenceNotFound", func(t *testing.T) { - imageRef := testRegistry.ImageRefForTest(t, "") + resp, err := dockerClient.ContainerInspect(t.Context(), container.ID) + require.NoError(t, err, "Failed to inspect container") + require.Equal(t, expected, resp) + }) - s.assertNoImageExists(t, imageRef) + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry - // so we handle other failure cases, like failed auth, unknown tag, and unknown repo - require.Error(t, err, "Failed to pull image %q", imageRef) - assert.ErrorIs(t, err, &command.NotFoundError{Object: "manifest", Ref: imageRef}) - assert.Nil(t, resp, "inspect response should be nil") + _, err := dockerClient.ContainerInspect(t.Context(), "containerid-that-does-not-exist") + require.ErrorIs(t, err, &command.NotFoundError{}) + }) }) - t.Run("InvalidAuth", func(t *testing.T) { - t.Skip("skip auth tests until we're using the docker engine since we can't set auth on the host without side effects") - imageRef := testRegistry.ImageRefForTest(t, "") - - s.assertNoImageExists(t, imageRef) + t.Run("ContainerLogs", func(t *testing.T) { + t.Parallel() - resp, err := s.dockerClient.Pull(t.Context(), imageRef, false) - // TODO[md]: this might not be the right check. we probably want to wrap the error from the registry - // so we handle other failure cases, like failed auth, unknown tag, and unknown repo - require.Error(t, err, "Failed to pull image %q", imageRef) - assert.ErrorContains(t, err, "failed to resolve reference") - assert.Nil(t, resp, "inspect response should be nil") + t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + // print "line $i" N times then exit, where $i is the line number + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"$i\"; sleep 1; done"), + // testcontainers.WithConfigModifier(func(config *container.Config) { + // config.Tty = true + // }), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + + assert.Equal(t, "1\n2\n3\n4\n5\n", buf.String()) + }) + + t.Run("ContainerAlreadyStopped", func(t *testing.T) { + t.Parallel() + + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 3); do echo \"$i\"; sleep 0.1; done"), + testcontainers.WithWaitStrategy(wait.ForExit()), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + state, err := container.State(t.Context()) + require.NoError(t, err, "Failed to get container state") + assert.Equal(t, state.Running, false) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + + assert.Equal(t, "1\n2\n3\n", buf.String()) + }) + + t.Run("TTY and non-TTY streams match", func(t *testing.T) { + t.Parallel() + + runContainer := func(tty bool) string { + container, err := testcontainers.Run( + t.Context(), + testRegistry.ImageRef("alpine:latest"), + // print "line $i" N times then exit, where $i is the line number + testcontainers.WithCmd("sh", "-c", "for i in $(seq 1 5); do echo \"$i\"; sleep 0.1; done"), + testcontainers.WithConfigModifier(func(config *container.Config) { + config.Tty = tty + }), + ) + require.NoError(t, err, "Failed to run container") + defer testcontainers.CleanupContainer(t, container) + + var buf bytes.Buffer + err = dockerClient.ContainerLogs(t.Context(), container.ID, &buf) + require.NoError(t, err, "Failed to get container logs") + return buf.String() + } + + ttyOutput := runContainer(true) + nonTtyOutput := runContainer(false) + + // TTY uses CRLF for line endings, non-TTY uses LF. replace \r\n with \n so they match + ttyOutput = strings.ReplaceAll(ttyOutput, "\r\n", "\n") + + assert.Equal(t, ttyOutput, nonTtyOutput, "TTY and non-TTY streams should match after normalizing line endings") + }) + + t.Run("ContainerDoesNotExist", func(t *testing.T) { + t.Parallel() + + err := dockerClient.ContainerLogs(t.Context(), "containerid-that-does-not-exist", &bytes.Buffer{}) + require.ErrorIs(t, err, &command.NotFoundError{}) + }) }) -} -func (s *DockerClientSuite) runContainerStopTests(t *testing.T) { - t.Run("ContainerExistsAndIsRunning", func(t *testing.T) { + t.Run("Push", func(t *testing.T) { t.Parallel() - container, err := testcontainers.Run( - t.Context(), - "docker.io/library/busybox:latest", - testcontainers.WithCmd("sleep", "5000"), - ) - defer testcontainers.CleanupContainer(t, container) - require.NoError(t, err, "Failed to run container") + t.Run("valid image, valid registry", func(t *testing.T) { + t.Parallel() - err = s.dockerClient.ContainerStop(t.Context(), container.ID) - require.NoError(t, err, "Failed to stop container %q", container.ID) + ref := dockertest.NewRef(t).WithRegistry(testRegistry.RegistryHost()) - state, err := container.State(t.Context()) - require.NoError(t, err, "Failed to get container state") - assert.Equal(t, state.Running, false) - }) + dockerHelper.ImageFixture(t, "alpine", ref.String()) - t.Run("ContainerExistsAndIsNotRunning", func(t *testing.T) { - t.Parallel() + err := dockerClient.Push(t.Context(), ref.String()) + require.NoError(t, err) + assert.NoError(t, testRegistry.ImageExists(t, ref.String())) + }) - container, err := testcontainers.GenericContainer(t.Context(), - testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "docker.io/library/busybox:latest", - Cmd: []string{"sleep", "5000"}, - }, - Started: false, - }, - ) - defer testcontainers.CleanupContainer(t, container) - containerID := container.GetContainerID() - require.NoError(t, err, "Failed to create container") - - err = s.dockerClient.ContainerStop(t.Context(), containerID) - require.NoError(t, err, "Failed to stop container %q", containerID) - - state, err := container.State(t.Context()) - require.NoError(t, err, "Failed to get container state") - assert.Equal(t, state.Running, false) - }) + t.Run("non-existent registry", func(t *testing.T) { + t.Parallel() - t.Run("ContainerDoesNotExist", func(t *testing.T) { - t.Parallel() + // start a local tcp server that immediately closes connections + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + conn.Close() + } + }() + + // Create a reference to the mock registry + ref := dockertest.NewRef(t).WithRegistry(listener.Addr().String()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) - err := s.dockerClient.ContainerStop(t.Context(), "containerid-that-does-not-exist") - require.ErrorIs(t, err, &command.NotFoundError{}) - require.ErrorContains(t, err, "container not found") + // Try to push to the mock registry + err = dockerClient.Push(t.Context(), ref.String()) + require.Error(t, err, "Push should fail with unreachable registry") + + // error message varies between dev and CI host environments, cover them all... + assert.Condition(t, func() bool { + msg := err.Error() + return strings.Contains(msg, "connection refused") || strings.Contains(msg, "EOF") + }, "Error should indicate registry is unreachable") + }) + + t.Run("missing image", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(testRegistry.RegistryHost()) + + err := dockerClient.Push(t.Context(), ref.String()) + assertNotFoundError(t, err, ref.String(), "tag") + }) + + t.Run("registry with authentication", func(t *testing.T) { + t.Parallel() + + if _, ok := dockerClient.(*DockerCommand); ok { + t.Skip("skipping auth tests for docker command client since we can't set auth on the host without side effects") + } + + authReg := registry_testhelpers.StartTestRegistry(t, registry_testhelpers.WithAuth("testuser", "testpass")) + + t.Run("correct credentials", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + // create a new client with the correct auth config + authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + Username: "testuser", + Password: "testpass", + ServerAddress: authReg.RegistryHost(), + })) + require.NoError(t, err) + + err = authClient.Push(t.Context(), ref.String()) + require.NoError(t, err, "Failed to push image to auth registry") + assert.NoError(t, authReg.ImageExists(t, ref.String())) + }) + + t.Run("missing auth", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + // use root client which doesn't have auth setup + err := dockerClient.Push(t.Context(), ref.String()) + require.ErrorIs(t, err, command.ErrAuthorizationFailed) + }) + + t.Run("incorrect auth", func(t *testing.T) { + t.Parallel() + + ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) + dockerHelper.ImageFixture(t, "alpine", ref.String()) + + authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + Username: "testuser", + Password: "wrongpass", + ServerAddress: authReg.RegistryHost(), + })) + require.NoError(t, err) + + err = authClient.Push(t.Context(), ref.String()) + require.ErrorIs(t, err, command.ErrAuthorizationFailed) + }) + + t.Run("correct credentials, not authorized", func(t *testing.T) { + t.Skip("skipping until the registry supports repo authorizations") + }) + }) }) } + +func assertImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + t.Helper() + + inspect, err := dockerClient.Inspect(t.Context(), imageRef) + assert.NoError(t, err, "Failed to inspect image %q", imageRef) + assert.NotNil(t, inspect, "Image should exist") +} + +func assertNoImageExists(t *testing.T, dockerClient command.Command, imageRef string) { + t.Helper() + + inspect, err := dockerClient.Inspect(t.Context(), imageRef) + assert.ErrorIs(t, err, &command.NotFoundError{}, "Image should not exist") + assert.Nil(t, inspect, "Image should not exist") +} + +func assertNotFoundError(t *testing.T, err error, ref string, object string) { + t.Helper() + + var notFoundErr *command.NotFoundError + require.ErrorAs(t, err, ¬FoundErr, "should be a not found error") + require.Equal(t, ref, notFoundErr.Ref, "ref should match") + require.Equal(t, object, notFoundErr.Object, "object should match") +} diff --git a/pkg/docker/dockertest/helper_client.go b/pkg/docker/dockertest/helper_client.go index 5e24c9e756..cd6f7c49ab 100644 --- a/pkg/docker/dockertest/helper_client.go +++ b/pkg/docker/dockertest/helper_client.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "slices" + "sync" "testing" "github.com/docker/docker/api/types/container" @@ -41,17 +42,33 @@ func NewHelperClient(t testing.TB) *HelperClient { t.Skip("Docker daemon is not running") } + helper := &HelperClient{ + Client: cli, + fixtures: make(map[string]*imageFixture), + mu: &sync.Mutex{}, + } + t.Cleanup(func() { + for _, img := range helper.fixtures { + _, err := helper.Client.ImageRemove(context.Background(), img.imageID, image.RemoveOptions{Force: true, PruneChildren: true}) + if err != nil { + t.Logf("Warning: Failed to remove image %q: %v", img.imageID, err) + } + } + if err := cli.Close(); err != nil { t.Fatalf("Failed to close Docker client: %v", err) } }) - return &HelperClient{Client: cli} + return helper } type HelperClient struct { Client *client.Client + + mu *sync.Mutex + fixtures map[string]*imageFixture } func (c *HelperClient) Close() error { @@ -210,10 +227,13 @@ func (c *HelperClient) CleanupImage(t testing.TB, imageRef string) { t.Helper() t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), imageRef, image.RemoveOptions{ + _, err := c.Client.ImageRemove(context.Background(), imageRef, image.RemoveOptions{ Force: true, PruneChildren: true, }) + if err != nil { + t.Logf("Warning: Failed to remove image %q: %v", imageRef, err) + } }) } @@ -255,9 +275,32 @@ func (c *HelperClient) InspectContainer(t testing.TB, containerID string) *conta return &inspect } -func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { +func (c *HelperClient) ImageFixture(t testing.TB, name string, tag string) { + t.Helper() + fixture := c.loadImageFixture(t, name) + + t.Logf("Tagging image fixture %q with %q", fixture.ref, tag) + if err := c.Client.ImageTag(t.Context(), fixture.imageID, tag); err != nil { + require.NoError(t, err, "Failed to tag image %q with %q: %v", fixture.ref, tag, err) + } + // remove the image when the test is done + t.Cleanup(func() { + _, _ = c.Client.ImageRemove(context.Background(), tag, image.RemoveOptions{Force: true}) + }) +} + +func (c *HelperClient) loadImageFixture(t testing.TB, name string) *imageFixture { t.Helper() + c.mu.Lock() + defer c.mu.Unlock() + + ref := fmt.Sprintf("cog-test-fixture:%s", name) + + if fixture, ok := c.fixtures[ref]; ok { + return fixture + } + // Get the path of the current file _, filename, _, ok := runtime.Caller(0) if !ok { @@ -270,7 +313,6 @@ func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { // Construct the path to the fixture fixturePath := filepath.Join(dir, "testdata", name+".tar") - ref := fmt.Sprintf("cog-test-fixture:%s", name) t.Logf("Loading image fixture %q from %s", ref, fixturePath) f, err := os.Open(fixturePath) @@ -280,22 +322,23 @@ func (c *HelperClient) LoadImageFixture(t testing.TB, name string, tag string) { l, err := c.Client.ImageLoad(t.Context(), f) require.NoError(t, err, "Failed to load fixture %q", name) defer l.Body.Close() - _, err = io.Copy(os.Stdout, l.Body) + _, err = io.Copy(os.Stderr, l.Body) require.NoError(t, err, "Failed to copy fixture %q", name) - // remove the image when the test is done - t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), ref, image.RemoveOptions{}) - }) + inspect, err := c.Client.ImageInspect(t.Context(), ref) + require.NoError(t, err, "Failed to inspect image %q", ref) - if tag != "" { - t.Logf("Tagging image fixture %q with %q", ref, tag) - if err := c.Client.ImageTag(t.Context(), ref, tag); err != nil { - require.NoError(t, err, "Failed to tag image %q with %q: %v", ref, tag, err) - } - // remove the image when the test is done - t.Cleanup(func() { - _, _ = c.Client.ImageRemove(context.Background(), tag, image.RemoveOptions{}) - }) + fixture := &imageFixture{ + ref: ref, + imageID: inspect.ID, } + + c.fixtures[ref] = fixture + + return fixture +} + +type imageFixture struct { + imageID string + ref string } diff --git a/pkg/docker/dockertest/ref.go b/pkg/docker/dockertest/ref.go new file mode 100644 index 0000000000..4b7d0fe9e3 --- /dev/null +++ b/pkg/docker/dockertest/ref.go @@ -0,0 +1,82 @@ +package dockertest + +import ( + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/stretchr/testify/require" +) + +type Ref struct { + t *testing.T + ref name.Reference +} + +func NewRef(t *testing.T) Ref { + t.Helper() + + repoName := strings.ToLower(t.Name()) + // Replace any characters that aren't valid in a docker image repo name with underscore + // Valid characters are: a-z, 0-9, ., _, -, / + repoName = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' || r == '/' { + return r + } + return '_' + }, repoName) + + ref, err := name.ParseReference(repoName, name.WithDefaultRegistry("")) + require.NoError(t, err, "Failed to create reference for test") + + return Ref{t: t, ref: ref} +} + +func (r Ref) WithTag(tagName string) Ref { + tagRef := r.ref.Context().Tag(tagName) + return Ref{t: r.t, ref: tagRef} +} + +func (r Ref) WithDigest(digest string) Ref { + digestRef := r.ref.Context().Digest(digest) + return Ref{t: r.t, ref: digestRef} +} + +func (r Ref) WithRegistry(registry string) Ref { + reg, err := name.NewRegistry(registry) + require.NoError(r.t, err, "Failed to create registry for test") + + repo := r.ref.Context() + repo.Registry = reg + var newRef name.Reference + switch r.ref.(type) { + case name.Tag: + newRef = repo.Tag(r.ref.Identifier()) + case name.Digest: + newRef = repo.Digest(r.ref.Identifier()) + default: + require.Fail(r.t, "Unsupported reference type") + } + + return Ref{t: r.t, ref: newRef} +} + +func (r Ref) WithoutRegistry() Ref { + repo := r.ref.Context() + repo.Registry = name.Registry{} + var newRef name.Reference + switch r.ref.(type) { + case name.Tag: + newRef = repo.Tag(r.ref.Identifier()) + case name.Digest: + newRef = repo.Digest(r.ref.Identifier()) + default: + require.Fail(r.t, "Unsupported reference type") + } + + return Ref{t: r.t, ref: newRef} +} + +func (r Ref) String() string { + return r.ref.Name() +} diff --git a/pkg/docker/dockertest/ref_test.go b/pkg/docker/dockertest/ref_test.go new file mode 100644 index 0000000000..754b12dec8 --- /dev/null +++ b/pkg/docker/dockertest/ref_test.go @@ -0,0 +1,24 @@ +package dockertest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRef(t *testing.T) { + ref := NewRef(t) + assert.Equal(t, "testref:latest", ref.String()) + + ref = ref.WithTag("v2") + assert.Equal(t, "testref:v2", ref.String()) + + ref = ref.WithRegistry("r8.im") + assert.Equal(t, "r8.im/testref:v2", ref.String()) + + ref = ref.WithoutRegistry() + assert.Equal(t, "testref:v2", ref.String()) + + ref = ref.WithDigest("sha256:71859b0c62df47efaeae4f93698b56a8dddafbf041778fd668bbd1ab45a864f8") + assert.Equal(t, "testref@sha256:71859b0c62df47efaeae4f93698b56a8dddafbf041778fd668bbd1ab45a864f8", ref.String()) +} diff --git a/pkg/registry_testhelpers/registry_container.go b/pkg/registry_testhelpers/registry_container.go index c64e0e2094..bc0cfb3521 100644 --- a/pkg/registry_testhelpers/registry_container.go +++ b/pkg/registry_testhelpers/registry_container.go @@ -12,10 +12,17 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/registry" "github.com/testcontainers/testcontainers-go/wait" + "golang.org/x/crypto/bcrypt" + + dockerregistry "github.com/docker/docker/api/types/registry" "github.com/replicate/cog/pkg/util" ) @@ -25,23 +32,26 @@ import ( // that can be used to inspect the registry and generate absolute image references. It will // automatically be cleaned when the test finishes. // This is safe to run concurrently across multiple tests. -func StartTestRegistry(t *testing.T) *RegistryContainer { +func StartTestRegistry(t *testing.T, opts ...Option) *RegistryContainer { t.Helper() + options := &options{} + for _, opt := range opts { + opt(options) + } + _, filename, _, _ := runtime.Caller(0) testdataDir := filepath.Join(filepath.Dir(filename), "testdata", "docker") - registryContainer, err := registry.Run( - t.Context(), - "registry:3", + containerCustomizers := []testcontainers.ContainerCustomizer{ testcontainers.WithFiles(testcontainers.ContainerFile{ HostFilePath: testdataDir, ContainerFilePath: "/var/lib/registry/", FileMode: 0o755, }), testcontainers.WithWaitStrategy( - wait.ForHTTP("/v2/").WithPort("5000/tcp"). - WithStartupTimeout(10*time.Second), + wait.ForHTTP("/").WithPort("5000/tcp"). + WithStartupTimeout(10 * time.Second), ), testcontainers.WithHostConfigModifier(func(hostConfig *container.HostConfig) { // docker only considers localhost:1 through localhost:9999 as insecure. testcontainers @@ -53,15 +63,33 @@ func StartTestRegistry(t *testing.T) *RegistryContainer { nat.Port("5000/tcp"): {{HostIP: "0.0.0.0", HostPort: strconv.Itoa(port)}}, } }), + } + + if options.auth != nil { + htpasswd, err := generateHtpasswd(options.auth.Username, options.auth.Password) + require.NoError(t, err) + containerCustomizers = append(containerCustomizers, + registry.WithHtpasswd(htpasswd), + ) + } + + registryContainer, err := registry.Run( + t.Context(), + "registry:3", + containerCustomizers..., ) defer testcontainers.CleanupContainer(t, registryContainer) require.NoError(t, err, "Failed to start registry container") - return &RegistryContainer{Container: registryContainer} + return &RegistryContainer{ + Container: registryContainer, + options: options, + } } type RegistryContainer struct { Container *registry.RegistryContainer + options *options } func (c *RegistryContainer) ImageRef(ref string) string { @@ -75,3 +103,59 @@ func (c *RegistryContainer) ImageRefForTest(t *testing.T, label string) string { repo := strings.ToLower(t.Name()) return c.ImageRef(fmt.Sprintf("%s:%s", repo, label)) } + +func (c *RegistryContainer) CloneRepo(t *testing.T, existingRepo, newRepo string) string { + existingRepo = c.ImageRef(existingRepo) + newRepo = c.ImageRef(newRepo) + + err := crane.CopyRepository(existingRepo, newRepo) + require.NoError(t, err, "Failed to clone repo %q to %q", existingRepo, newRepo) + return newRepo +} + +func (c *RegistryContainer) CloneRepoForTest(t *testing.T, repo string) string { + return c.CloneRepo(t, repo, strings.ToLower(t.Name())) +} + +func (c *RegistryContainer) ImageExists(t *testing.T, ref string) error { + parsedRef, err := name.ParseReference(ref, name.WithDefaultRegistry(c.RegistryHost())) + require.NoError(t, err) + + var opts []remote.Option + + if c.options.auth != nil { + opts = append(opts, remote.WithAuth(authn.FromConfig(authn.AuthConfig{ + Username: c.options.auth.Username, + Password: c.options.auth.Password, + }))) + } + _, err = remote.Head(parsedRef, opts...) + return err +} + +func (c *RegistryContainer) RegistryHost() string { + return c.Container.RegistryName +} + +type Option func(*options) + +func WithAuth(username, password string) func(*options) { + return func(o *options) { + o.auth = &dockerregistry.AuthConfig{ + Username: username, + Password: password, + } + } +} + +type options struct { + auth *dockerregistry.AuthConfig +} + +func generateHtpasswd(username, password string) (string, error) { + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", username, string(hash)), nil +} From 9246b18f79a3f85e5e3e04c2ccb12e305ebce657 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:35:53 -0600 Subject: [PATCH 34/43] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b6e6a0c711..4bc05a867d 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 github.com/xeonx/timeago v1.0.0-rc5 + golang.org/x/crypto v0.37.0 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 golang.org/x/sync v0.14.0 golang.org/x/sys v0.33.0 @@ -272,7 +273,6 @@ require ( go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.39.0 // indirect From 9e51b896a0ccf3fc1283767eb57e91f052c991e9 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:52:46 -0600 Subject: [PATCH 35/43] fix for merge flub and correct new client func name --- pkg/docker/docker_client_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index e3bcb93c51..eb85c45290 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -26,9 +26,6 @@ func TestDockerClient(t *testing.T) { client := NewDockerCommand() runDockerClientTests(t, client) -} - } - } func runDockerClientTests(t *testing.T, dockerClient command.Command) { @@ -364,7 +361,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { dockerHelper.ImageFixture(t, "alpine", ref.String()) // create a new client with the correct auth config - authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + authClient, err := NewClient(t.Context(), WithAuthConfig(registry.AuthConfig{ Username: "testuser", Password: "testpass", ServerAddress: authReg.RegistryHost(), @@ -393,7 +390,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { ref := dockertest.NewRef(t).WithRegistry(authReg.RegistryHost()) dockerHelper.ImageFixture(t, "alpine", ref.String()) - authClient, err := NewAPIClient(t.Context(), WithAuthConfig(registry.AuthConfig{ + authClient, err := NewClient(t.Context(), WithAuthConfig(registry.AuthConfig{ Username: "testuser", Password: "wrongpass", ServerAddress: authReg.RegistryHost(), From 2639c32fe5a3548ec0f91bbe8201f24e9d2817cf Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 16:59:01 -0600 Subject: [PATCH 36/43] remove some code that's only used in the sdk client --- pkg/docker/credentials.go | 62 --------------------------------------- 1 file changed, 62 deletions(-) diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go index 722dc99851..28b26332b9 100644 --- a/pkg/docker/credentials.go +++ b/pkg/docker/credentials.go @@ -12,7 +12,6 @@ import ( "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/config/types" - "github.com/docker/docker/api/types/registry" "github.com/replicate/cog/pkg/docker/command" "github.com/replicate/cog/pkg/util/console" @@ -45,67 +44,6 @@ func loadAuthFromConfig(conf *configfile.ConfigFile, registryHost string) (types return conf.AuthConfigs[registryHost], nil } -func loadRegistryAuths(ctx context.Context, registryHosts ...string) (map[string]registry.AuthConfig, error) { - conf := config.LoadDefaultConfigFile(os.Stderr) - - out := make(map[string]registry.AuthConfig) - - for _, host := range registryHosts { - console.Debugf("=== loadRegistryAuths %s", host) - // check the credentials store first if set - if conf.CredentialsStore != "" { - console.Debugf("=== loadRegistryAuths %s: credentials store set", host) - credsHelper, err := loadAuthFromCredentialsStore(ctx, conf.CredentialsStore, host) - if err != nil { - console.Debugf("=== loadRegistryAuths %s: error loading credentials store: %s", host, err) - return nil, err - } - console.Debugf("=== loadRegistryAuths %s: credentials store loaded", host) - out[host] = registry.AuthConfig{ - Username: credsHelper.Username, - Password: credsHelper.Secret, - ServerAddress: host, - } - continue - } - - // next, check if the auth config exists in the config file - if auth, ok := conf.AuthConfigs[host]; ok { - console.Debugf("=== loadRegistryAuths %s: auth config found in config file", host) - out[host] = registry.AuthConfig{ - Username: auth.Username, - Password: auth.Password, - Auth: auth.Auth, - Email: auth.Email, - ServerAddress: host, - IdentityToken: auth.IdentityToken, - RegistryToken: auth.RegistryToken, - } - continue - } - - console.Debugf("=== loadRegistryAuths %s: no auth config found", host) - - // TODO[md]: should we error here!? probably not... - } - - return out, nil -} - -// func loadRegistryAuths(conf *configfile.ConfigFile, registryHosts ...string) (map[string]types.AuthConfig, error) { -// out := make(map[string]types.AuthConfig) -// for _, host := range registryHosts { -// auth, ok := conf.AuthConfigs[host] -// if !ok { -// // TODO[md]: should we return an error here or just carry on? -// return nil, fmt.Errorf("no auth config found for registry host %s", host) -// } -// out[host] = auth -// } - -// return out, nil -// } - func loadAuthFromCredentialsStore(ctx context.Context, credsStore string, registryHost string) (*CredentialHelperInput, error) { var out strings.Builder binary := dockerCredentialBinary(credsStore) From 59d8a734dfb52718f58c03ff59afc5f105457c23 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 17:36:54 -0600 Subject: [PATCH 37/43] lint fix --- pkg/docker/docker_client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index 4330b2227f..cae80503ee 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -331,7 +331,7 @@ func runDockerClientTests(t *testing.T, dockerClient command.Command) { } }() - // Create a reference to the mock registry + // Create a reference to the mock registry ref := dockertest.NewRef(t).WithRegistry(listener.Addr().String()) dockerHelper.ImageFixture(t, "alpine", ref.String()) From a4b5d2a6f46235344dcfee7e01aa7c563c7ca333 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 17:43:14 -0600 Subject: [PATCH 38/43] remove debug prints --- pkg/docker/build_secrets.go | 10 +--------- pkg/docker/buildkit.go | 4 ---- pkg/docker/credentials.go | 2 -- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/pkg/docker/build_secrets.go b/pkg/docker/build_secrets.go index b05c3b03dd..27aaa2bf38 100644 --- a/pkg/docker/build_secrets.go +++ b/pkg/docker/build_secrets.go @@ -19,18 +19,10 @@ func ParseSecretsFromHost(workingDir string, secrets []string) (secrets.SecretSt if err != nil { return nil, err } - fmt.Println("src", src) sources = append(sources, *src) } - fmt.Println("sources", sources) - store, err := secretsprovider.NewStore(sources) - if err != nil { - return nil, err - } - fmt.Println("store", store) - - return store, nil + return secretsprovider.NewStore(sources) } func parseSecretFromHost(workingDir, secret string) (*secretsprovider.Source, error) { diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go index 38f6082b11..8f4c706648 100644 --- a/pkg/docker/buildkit.go +++ b/pkg/docker/buildkit.go @@ -37,10 +37,6 @@ func solveOptFromImageOptions(buildDir string, opts command.ImageBuildOptions) ( return buildkitclient.SolveOpt{}, err } - fmt.Printf("workingdir %q\ncontextdir %q\n", opts.WorkingDir, opts.ContextDir) - - // Secrets []string - // first, configure the frontend, in this case, dockerfile.v0 frontendAttrs := map[string]string{ // filename is the path to the Dockerfile within the "dockerfile" LocalDir context diff --git a/pkg/docker/credentials.go b/pkg/docker/credentials.go index e59778d694..1ac5c0ed8d 100644 --- a/pkg/docker/credentials.go +++ b/pkg/docker/credentials.go @@ -85,8 +85,6 @@ func loadRegistryAuths(ctx context.Context, registryHosts ...string) (map[string } console.Debugf("=== loadRegistryAuths %s: no auth config found", host) - - // TODO[md]: should we error here!? probably not... } return out, nil From 5c43f6cc238ee63eb8304844ebea61ecf48d613a Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 17:45:03 -0600 Subject: [PATCH 39/43] bring back auto remove --- pkg/docker/api_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/docker/api_client.go b/pkg/docker/api_client.go index daecbb7645..7fa128e5ee 100644 --- a/pkg/docker/api_client.go +++ b/pkg/docker/api_client.go @@ -354,7 +354,7 @@ func (c *apiClient) containerRun(ctx context.Context, options command.RunOptions hostCfg := &container.HostConfig{ // always remove container after it exits - // AutoRemove: true, + AutoRemove: true, // https://github.com/pytorch/pytorch/issues/2244 // https://github.com/replicate/cog/issues/1293 ShmSize: 6 * 1024 * 1024 * 1024, // 6GB From af73d3db016bfaa7218a833091e1d8b44e3e6332 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 17:47:04 -0600 Subject: [PATCH 40/43] fix comments --- pkg/docker/buildkit.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/docker/buildkit.go b/pkg/docker/buildkit.go index 8f4c706648..d961f0b777 100644 --- a/pkg/docker/buildkit.go +++ b/pkg/docker/buildkit.go @@ -170,14 +170,13 @@ func newBuildkitAuthProvider(registryHosts ...string) session.Attachable { registryHosts: sync.OnceValues(func() (map[string]registry.AuthConfig, error) { return loadRegistryAuths(context.Background(), registryHosts...) }), + // TODO[md]: here's where we'd set the token from config rather than fetching from the credentials helper // token: token, } } type buildkitAuthProvider struct { registryHosts func() (map[string]registry.AuthConfig, error) - // auths map[string]registry.AuthConfig - // token string } func (ap *buildkitAuthProvider) Register(server *grpc.Server) { From 74287d126c0bd0a4933e05c59478b05c965d5201 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Fri, 16 May 2025 20:42:15 -0600 Subject: [PATCH 41/43] set env var so subtests pickup the right client --- pkg/docker/docker_client_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/docker/docker_client_test.go b/pkg/docker/docker_client_test.go index cae80503ee..b66c079bac 100644 --- a/pkg/docker/docker_client_test.go +++ b/pkg/docker/docker_client_test.go @@ -24,6 +24,8 @@ func TestDockerClient(t *testing.T) { t.Skip("skipping docker client tests in short mode") } + t.Setenv("COG_DOCKER_SDK_CLIENT", "0") + client := NewDockerCommand() runDockerClientTests(t, client) } @@ -33,6 +35,8 @@ func TestDockerAPIClient(t *testing.T) { t.Skip("skipping docker client tests in short mode") } + t.Setenv("COG_DOCKER_SDK_CLIENT", "1") + apiClient, err := NewAPIClient(t.Context()) require.NoError(t, err, "Failed to create docker api client") runDockerClientTests(t, apiClient) From cf167abf25c76df85f6e0841d73f4685270f3d5f Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 21 May 2025 11:32:54 -0600 Subject: [PATCH 42/43] fix merge issue --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b7742e93d..c2a331a5a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -151,6 +151,7 @@ jobs: run: uv pip install --system tox tox-uv - name: Test run: make test-integration + env: COG_DOCKER_SDK_CLIENT: ${{ matrix.new_docker_api_client }} release: From 59efcf099e76ad75b156eb9789c0b18707b16aa8 Mon Sep 17 00:00:00 2001 From: Michael Dwan Date: Wed, 21 May 2025 11:55:55 -0600 Subject: [PATCH 43/43] hack to pass required checks --- .github/workflows/ci.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2a331a5a6..34b1a0f6aa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,9 +119,19 @@ jobs: - name: Test run: python -Im tox run --installpkg "$COG_WHEEL" -e ${{ env.TOX_PYTHON }}-pydantic${{ matrix.pydantic }}-tests + # TODO[md]: This is a gross hack, remove once this is sorted out: https://github.com/replicate/cog/pull/2353 # cannot run this on mac due to licensing issues: https://github.com/actions/virtual-environments/issues/2150 test-integration: name: "Test integration" + needs: test-integration-matrix + runs-on: ubuntu-latest + steps: + - name: Check test status + run: echo "All tests passed successfully!" + + # cannot run this on mac due to licensing issues: https://github.com/actions/virtual-environments/issues/2150 + test-integration-matrix: + name: "Test integration Matrix" needs: build-python runs-on: ubuntu-latest-16-cores strategy: