From e1712761414158e1681275ff7f2a71958a522636 Mon Sep 17 00:00:00 2001 From: Will Sackfield Date: Wed, 16 Apr 2025 17:40:17 -0400 Subject: [PATCH 1/4] Add zsh integration test --- .../fixtures/zsh-package/cog.yaml | 7 ++++++ .../fixtures/zsh-package/predict.py | 8 +++++++ .../fixtures/zsh-package/requirements.txt | 1 + .../test_integration/test_predict.py | 22 +++++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 test-integration/test_integration/fixtures/zsh-package/cog.yaml create mode 100644 test-integration/test_integration/fixtures/zsh-package/predict.py create mode 100644 test-integration/test_integration/fixtures/zsh-package/requirements.txt diff --git a/test-integration/test_integration/fixtures/zsh-package/cog.yaml b/test-integration/test_integration/fixtures/zsh-package/cog.yaml new file mode 100644 index 0000000000..8e51719c91 --- /dev/null +++ b/test-integration/test_integration/fixtures/zsh-package/cog.yaml @@ -0,0 +1,7 @@ +build: + python_version: "3.12" + fast: true + system_packages: + - "zsh" + python_requirements: requirements.txt +predict: "predict.py:Predictor" diff --git a/test-integration/test_integration/fixtures/zsh-package/predict.py b/test-integration/test_integration/fixtures/zsh-package/predict.py new file mode 100644 index 0000000000..27d335ee27 --- /dev/null +++ b/test-integration/test_integration/fixtures/zsh-package/predict.py @@ -0,0 +1,8 @@ +from cog import BasePredictor + +import os + + +class Predictor(BasePredictor): + def predict(self) -> str: + return "hello " + ",".join(os.listdir("/bin")) diff --git a/test-integration/test_integration/fixtures/zsh-package/requirements.txt b/test-integration/test_integration/fixtures/zsh-package/requirements.txt new file mode 100644 index 0000000000..7e820f150e --- /dev/null +++ b/test-integration/test_integration/fixtures/zsh-package/requirements.txt @@ -0,0 +1 @@ +torch==2.5.0 diff --git a/test-integration/test_integration/test_predict.py b/test-integration/test_integration/test_predict.py index 960622bd47..edb21cab84 100644 --- a/test-integration/test_integration/test_predict.py +++ b/test-integration/test_integration/test_predict.py @@ -432,3 +432,25 @@ def test_predict_optional_project(tmpdir_factory): # stdout should be clean without any log messages so it can be piped to other commands assert result.returncode == 0 assert result.stdout == "hello No One\n" + + +def test_predict_zsh_package(docker_image): + project_dir = Path(__file__).parent / "fixtures/zsh-package" + build_process = subprocess.run( + ["cog", "build", "-t", docker_image], + cwd=project_dir, + capture_output=True, + ) + assert build_process.returncode == 0 + + result = subprocess.run( + ["cog", "predict", "--debug", docker_image], + cwd=project_dir, + check=True, + capture_output=True, + text=True, + timeout=DEFAULT_TIMEOUT, + ) + # stdout should be clean without any log messages so it can be piped to other commands + assert result.returncode == 0 + assert result.stdout == "hello No One\n" From 2468bedab7afa6171c581f26ed19812aa70ffae9 Mon Sep 17 00:00:00 2001 From: Will Sackfield Date: Wed, 16 Apr 2025 17:42:11 -0400 Subject: [PATCH 2/4] Extract then copy the apt package * Instead of extracting to root, extract to tmp then copy over to root * This preserves symlinked directories --- pkg/dockerfile/fast_generator.go | 2 +- pkg/dockerfile/fast_generator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dockerfile/fast_generator.go b/pkg/dockerfile/fast_generator.go index a703778ac6..09f3c9304b 100644 --- a/pkg/dockerfile/fast_generator.go +++ b/pkg/dockerfile/fast_generator.go @@ -334,7 +334,7 @@ func (g *FastGenerator) installApt(lines []string, aptTarFile string) ([]string, // Install apt packages if aptTarFile != "" { - lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" tar --no-overwrite-dir -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" -C /") + lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" mkdir /tmp/extract && tar -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" --no-overwrite-dir -C /tmp/extract && cp -a /tmp/extract / && rm -rf /tmp/extract") } return lines, nil } diff --git a/pkg/dockerfile/fast_generator_test.go b/pkg/dockerfile/fast_generator_test.go index 94c8637a2b..d70af59581 100644 --- a/pkg/dockerfile/fast_generator_test.go +++ b/pkg/dockerfile/fast_generator_test.go @@ -227,7 +227,7 @@ func TestAptInstall(t *testing.T) { dockerfile, err := generator.GenerateDockerfileWithoutSeparateWeights() require.NoError(t, err) dockerfileLines := strings.Split(dockerfile, "\n") - require.Equal(t, "RUN --mount=from=apt,target=/buildtmp tar --no-overwrite-dir -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" -C /", dockerfileLines[5]) + require.Equal(t, "RUN --mount=from=apt,target=/buildtmp mkdir /tmp/extract && tar -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" --no-overwrite-dir -C /tmp/extract && cp -a /tmp/extract / && rm -rf /tmp/extract", dockerfileLines[5]) } func TestValidateConfigWithBuildRunItems(t *testing.T) { From 34e10f937bd1735a9cd30020f4bf150b29a0f5b4 Mon Sep 17 00:00:00 2001 From: Will Sackfield Date: Thu, 17 Apr 2025 10:20:17 -0400 Subject: [PATCH 3/4] Use rsync to copy * Do not clobber symlinked dirs --- pkg/dockerfile/fast_generator.go | 2 +- pkg/dockerfile/fast_generator_test.go | 2 +- test-integration/test_integration/test_predict.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/dockerfile/fast_generator.go b/pkg/dockerfile/fast_generator.go index 09f3c9304b..11f78a74f0 100644 --- a/pkg/dockerfile/fast_generator.go +++ b/pkg/dockerfile/fast_generator.go @@ -334,7 +334,7 @@ func (g *FastGenerator) installApt(lines []string, aptTarFile string) ([]string, // Install apt packages if aptTarFile != "" { - lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" mkdir /tmp/extract && tar -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" --no-overwrite-dir -C /tmp/extract && cp -a /tmp/extract / && rm -rf /tmp/extract") + lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" mkdir /tmp/extract && tar -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" --no-overwrite-dir -C /tmp/extract && rsync -a --keep-dirlinks /tmp/extract/ / && rm -rf /tmp/extract") } return lines, nil } diff --git a/pkg/dockerfile/fast_generator_test.go b/pkg/dockerfile/fast_generator_test.go index d70af59581..383b537083 100644 --- a/pkg/dockerfile/fast_generator_test.go +++ b/pkg/dockerfile/fast_generator_test.go @@ -227,7 +227,7 @@ func TestAptInstall(t *testing.T) { dockerfile, err := generator.GenerateDockerfileWithoutSeparateWeights() require.NoError(t, err) dockerfileLines := strings.Split(dockerfile, "\n") - require.Equal(t, "RUN --mount=from=apt,target=/buildtmp mkdir /tmp/extract && tar -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" --no-overwrite-dir -C /tmp/extract && cp -a /tmp/extract / && rm -rf /tmp/extract", dockerfileLines[5]) + require.Equal(t, "RUN --mount=from=apt,target=/buildtmp mkdir /tmp/extract && tar -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" --no-overwrite-dir -C /tmp/extract && rsync -a --keep-dirlinks /tmp/extract/ / && rm -rf /tmp/extract", dockerfileLines[5]) } func TestValidateConfigWithBuildRunItems(t *testing.T) { diff --git a/test-integration/test_integration/test_predict.py b/test-integration/test_integration/test_predict.py index edb21cab84..2e26b4753f 100644 --- a/test-integration/test_integration/test_predict.py +++ b/test-integration/test_integration/test_predict.py @@ -453,4 +453,5 @@ def test_predict_zsh_package(docker_image): ) # stdout should be clean without any log messages so it can be piped to other commands assert result.returncode == 0 - assert result.stdout == "hello No One\n" + assert ",sh," in result.stdout + assert ",zsh," in result.stdout From 15639b32f74f13b4de925cc8de3395f3d27da71a Mon Sep 17 00:00:00 2001 From: Will Sackfield Date: Thu, 17 Apr 2025 14:25:17 -0400 Subject: [PATCH 4/4] Use --keep-directory-symlink instead --- pkg/dockerfile/fast_generator.go | 2 +- pkg/dockerfile/fast_generator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dockerfile/fast_generator.go b/pkg/dockerfile/fast_generator.go index 11f78a74f0..476bd56ecf 100644 --- a/pkg/dockerfile/fast_generator.go +++ b/pkg/dockerfile/fast_generator.go @@ -334,7 +334,7 @@ func (g *FastGenerator) installApt(lines []string, aptTarFile string) ([]string, // Install apt packages if aptTarFile != "" { - lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" mkdir /tmp/extract && tar -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" --no-overwrite-dir -C /tmp/extract && rsync -a --keep-dirlinks /tmp/extract/ / && rm -rf /tmp/extract") + lines = append(lines, "RUN --mount=from="+dockercontext.AptBuildContextName+",target="+buildTmpDir+" tar --keep-directory-symlink -xf \""+filepath.Join(buildTmpDir, aptTarFile)+"\" -C /") } return lines, nil } diff --git a/pkg/dockerfile/fast_generator_test.go b/pkg/dockerfile/fast_generator_test.go index 383b537083..264413e5ba 100644 --- a/pkg/dockerfile/fast_generator_test.go +++ b/pkg/dockerfile/fast_generator_test.go @@ -227,7 +227,7 @@ func TestAptInstall(t *testing.T) { dockerfile, err := generator.GenerateDockerfileWithoutSeparateWeights() require.NoError(t, err) dockerfileLines := strings.Split(dockerfile, "\n") - require.Equal(t, "RUN --mount=from=apt,target=/buildtmp mkdir /tmp/extract && tar -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" --no-overwrite-dir -C /tmp/extract && rsync -a --keep-dirlinks /tmp/extract/ / && rm -rf /tmp/extract", dockerfileLines[5]) + require.Equal(t, "RUN --mount=from=apt,target=/buildtmp tar --keep-directory-symlink -xf \"/buildtmp/apt.9a881b9b9f23849475296a8cd768ea1965bc3152df7118e60c145975af6aa58a.tar.zst\" -C /", dockerfileLines[5]) } func TestValidateConfigWithBuildRunItems(t *testing.T) {