Merge branch 'main' into fricounet/upstream/index-detect

Signed-off-by: Baptiste Girard-Carrabin <[email protected]>

Author: Fricounet

pkg/backend/s3.go

@@ -38,16 +38,18 @@ type S3Backend struct {
 	accessKeySecret    string
 	accessKeyID        string
 	forcePush          bool
+	checksumAlgorithm  types.ChecksumAlgorithm
 }
 
 type S3Config struct {
-	AccessKeyID     string `json:"access_key_id,omitempty"`
-	AccessKeySecret string `json:"access_key_secret,omitempty"`
-	Endpoint        string `json:"endpoint,omitempty"`
-	Scheme          string `json:"scheme,omitempty"`
-	BucketName      string `json:"bucket_name,omitempty"`
-	Region          string `json:"region,omitempty"`
-	ObjectPrefix    string `json:"object_prefix,omitempty"`
+	AccessKeyID       string  `json:"access_key_id,omitempty"`
+	AccessKeySecret   string  `json:"access_key_secret,omitempty"`
+	Endpoint          string  `json:"endpoint,omitempty"`
+	Scheme            string  `json:"scheme,omitempty"`
+	BucketName        string  `json:"bucket_name,omitempty"`
+	Region            string  `json:"region,omitempty"`
+	ObjectPrefix      string  `json:"object_prefix,omitempty"`
+	ChecksumAlgorithm *string `json:"checksum_algorithm,omitempty"`
 }
 
 func newS3Backend(rawConfig []byte, forcePush bool) (*S3Backend, error) {
@@ -67,6 +69,22 @@ func newS3Backend(rawConfig []byte, forcePush bool) (*S3Backend, error) {
 		return nil, fmt.Errorf("invalid S3 configuration: missing 'bucket_name' or 'region'")
 	}
 
+	var checksumAlgorithm types.ChecksumAlgorithm
+	if cfg.ChecksumAlgorithm == nil {
+		// Default to CRC32 checksum
+		checksumAlgorithm = types.ChecksumAlgorithmCrc32
+	} else if *cfg.ChecksumAlgorithm != "" {
+		for _, algorithm := range checksumAlgorithm.Values() {
+			if string(algorithm) == *cfg.ChecksumAlgorithm {
+				checksumAlgorithm = algorithm
+				break
+			}
+		}
+		if checksumAlgorithm == "" {
+			return nil, fmt.Errorf("invalid checksum algorithm: %s, supported algorithms: %v", *cfg.ChecksumAlgorithm, checksumAlgorithm.Values())
+		}
+	}
+
 	return &S3Backend{
 		objectPrefix:       cfg.ObjectPrefix,
 		bucketName:         cfg.BucketName,
@@ -75,6 +93,7 @@ func newS3Backend(rawConfig []byte, forcePush bool) (*S3Backend, error) {
 		accessKeySecret:    cfg.AccessKeySecret,
 		accessKeyID:        cfg.AccessKeyID,
 		forcePush:          forcePush,
+		checksumAlgorithm:  checksumAlgorithm,
 	}, nil
 }
 
@@ -144,7 +163,7 @@ func (b *S3Backend) Push(ctx context.Context, cs content.Store, desc ocispec.Des
 		Bucket:            aws.String(b.bucketName),
 		Key:               aws.String(blobObjectKey),
 		Body:              reader,
-		ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
+		ChecksumAlgorithm: b.checksumAlgorithm,
 	}); err != nil {
 		return errors.Wrap(err, "push blob to s3 backend")
 	}

pkg/backend/s3_test.go

@@ -9,6 +9,8 @@ package backend
 import (
 	"reflect"
 	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 func Test_newS3Backend(t *testing.T) {
@@ -42,6 +44,32 @@ func Test_newS3Backend(t *testing.T) {
 				region:             "us-east-1",
 				accessKeySecret:    "minio123",
 				accessKeyID:        "minio",
+				checksumAlgorithm:  types.ChecksumAlgorithmCrc32,
+			},
+			wantErr: false,
+		},
+		{
+			name: "test2, set checksum algorithm",
+			args: args{
+				rawConfig: []byte(`{
+					"endpoint": "localhost:9000",
+					"scheme": "http",
+					"bucket_name": "nydus",
+					"region": "us-east-1",
+					"object_prefix": "path/to/my-registry/",
+					"access_key_id": "minio",
+					"access_key_secret": "minio123",
+					"checksum_algorithm": "SHA256"
+				}`),
+			},
+			want: &S3Backend{
+				objectPrefix:       "path/to/my-registry/",
+				bucketName:         "nydus",
+				endpointWithScheme: "http://localhost:9000",
+				region:             "us-east-1",
+				accessKeySecret:    "minio123",
+				accessKeyID:        "minio",
+				checksumAlgorithm:  types.ChecksumAlgorithmSha256,
 			},
 			wantErr: false,
 		},

pkg/converter/constant.go

@@ -9,6 +9,7 @@ package converter
 const (
 	ManifestOSFeatureNydus    = "nydus.remoteimage.v1"
 	ManifestArtifactTypeNydus = "application/vnd.nydus.image.manifest.v1+json"
+  ManifestConfigNydus       = "application/vnd.nydus.image.config.v1+json"
 	MediaTypeNydusBlob        = "application/vnd.oci.image.layer.nydus.blob.v1"
 	BootstrapFileNameInLayer  = "image/image.boot"
 

pkg/converter/convert_unix.go

@@ -1041,6 +1041,12 @@ func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Desc
 	if err != nil {
 		return nil, errors.Wrap(err, "write image config")
 	}
+	// When manifests are merged, we need to put a special value for the config mediaType.
+	// This values must be one that containerd doesn't understand to ensure it doesn't try tu pull the nydus image
+	// but use the OCI one instead. And then if the nydus-snapshotter is used, it can pull the nydus image instead.
+	if opt.MergeManifest {
+		newConfigDesc.MediaType = ManifestConfigNydus
+	}
 	manifest.Config = *newConfigDesc
 	// Update the config gc label
 	manifestLabels[configGCLabelKey] = newConfigDesc.Digest.String()

pkg/converter/types.go

@@ -127,6 +127,9 @@ type MergeOption struct {
 	Encrypt Encrypter
 	// AppendFiles specifies the files that need to be appended to the bootstrap layer.
 	AppendFiles []File
+	// MergeManifest indicates that the resulting nydus manifest will be merged with the original
+	// OCI one into a single index manifest.
+	MergeManifest bool
 }
 
 type UnpackOption struct {