merge dev/oci feature to main

.github/workflows/build-and-push.yaml

@@ -170,6 +170,10 @@ jobs:
 
           rm -Rf $(ls . | grep -v config)
           rm -Rf .gitignore .dockerignore .github .git .yamllint.yaml
+
+          cat ./config/base/params.env
+          cat ./config/overlays/odh/params.env
+          cat ./config/overlays/rhoai/params.env
       # pysh to ci-manifest repo
       - uses: cpina/github-action-push-to-another-repository@main
         if: env.BUILD_CONTEXT == 'ci'

Dockerfile.lmes-job

@@ -3,6 +3,9 @@ FROM registry.access.redhat.com/ubi9/python-311@sha256:fccda5088dd13d2a3f2659e4c
 USER root
 RUN sed -i.bak 's/include-system-site-packages = false/include-system-site-packages = true/' /opt/app-root/pyvenv.cfg
 
+# required dependency for oci.py in lmes-job; put here for the `needs-lmes-build`, but already incorporated here: https://github.com/opendatahub-io/lm-evaluation-harness/blob/3c4dec006a4a096b546d60c3364c78acbe33cd48/Dockerfile.lmes-job#L16
+RUN dnf install -y skopeo && dnf clean all
+
 USER default
 WORKDIR /opt/app-root/src
 RUN mkdir /opt/app-root/src/hf_home && chmod g+rwx /opt/app-root/src/hf_home

api/lmes/v1alpha1/lmevaljob_types.go

@@ -400,6 +400,9 @@ type Outputs struct {
 	// Create an operator managed PVC
 	// +optional
 	PersistentVolumeClaimManaged *PersistentVolumeClaimManaged `json:"pvcManaged,omitempty"`
+	// Upload results to OCI registry
+	// +optional
+	OCISpec *OCISpec `json:"oci,omitempty"`
 }
 
 func (c *LMEvalContainer) GetSecurityContext() *corev1.SecurityContext {
@@ -463,6 +466,36 @@ type OfflineS3Spec struct {
 	CABundle  *corev1.SecretKeySelector `json:"caBundle,omitempty"`
 }
 
+type OCISpec struct {
+	// Registry URL (e.g., quay.io, registry.redhat.com)
+	Registry corev1.SecretKeySelector `json:"registry"`
+	// Repository path (e.g., myorg/evaluation-results)
+	// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._/-]*$`
+	Repository string `json:"repository"`
+	// Optional tag for the artifact (defaults to job name if not specified)
+	// +optional
+	// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`
+	// +kubebuilder:validation:MaxLength=128
+	Tag string `json:"tag,omitempty"`
+	// Subject for the OCI artifact
+	// +optional
+	// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._:/@-]*$`
+	// +kubebuilder:validation:MaxLength=255
+	Subject string `json:"subject,omitempty"`
+	// Username for registry authentication
+	// +optional
+	UsernameRef *corev1.SecretKeySelector `json:"username,omitempty"`
+	// Password for registry authentication
+	// +optional
+	PasswordRef *corev1.SecretKeySelector `json:"password,omitempty"`
+	// DockerConfigJson for registry authentication (alternative to username/password)
+	// +optional
+	DockerConfigJsonRef *corev1.SecretKeySelector `json:"dockerConfigJson,omitempty"`
+	// Whether to verify SSL certificates
+	// +optional
+	VerifySSL *bool `json:"verifySSL,omitempty"`
+}
+
 // OfflineStorageSpec defines the storage configuration for LMEvalJob's offline mode
 type OfflineStorageSpec struct {
 	PersistentVolumeClaimName *string        `json:"pvcName,omitempty"`
@@ -585,10 +618,22 @@ func (s *LMEvalJobSpec) HasOfflineS3() bool {
 	return s.Offline != nil && s.Offline.StorageSpec.S3Spec != nil
 }
 
+func (s *LMEvalJobSpec) HasOCIOutput() bool {
+	return s.Outputs != nil && s.Outputs.OCISpec != nil
+}
+
 func (s *OfflineS3Spec) HasCertificates() bool {
 	return s.CABundle != nil
 }
 
+func (s *OCISpec) HasUsernamePassword() bool {
+	return s.UsernameRef != nil && s.PasswordRef != nil
+}
+
+func (s *OCISpec) HasDockerConfigJson() bool {
+	return s.DockerConfigJsonRef != nil
+}
+
 // HasCustomOutput returns whether an LMEvalJobSpec defines custom outputs or not
 func (s *LMEvalJobSpec) HasCustomOutput() bool {
 	return s.Outputs != nil
@@ -604,6 +649,11 @@ func (o *Outputs) HasExistingPVC() bool {
 	return o.PersistentVolumeClaimName != nil
 }
 
+// HasOCI returns whether the outputs define OCI upload
+func (o *Outputs) HasOCI() bool {
+	return o != nil && o.OCISpec != nil
+}
+
 // LMEvalJobStatus defines the observed state of LMEvalJob
 type LMEvalJobStatus struct {
 	// Important: Run "make" to regenerate code after modifying this file

cmd/lmes_driver/main.go

@@ -60,6 +60,7 @@ var (
 	detectDevice        = flag.Bool("detect-device", false, "detect available device(s), CUDA or CPU")
 	commPort            = flag.Int("listen-port", driver.DefaultPort, "driver serves APIs on the port")
 	downloadAssetsS3    = flag.Bool("download-assets-s3", false, "Download assets from S3")
+	uploadToOCI         = flag.Bool("upload-to-oci", false, "Upload results to OCI registry")
 	customTaskGitURL    = flag.String("custom-task-git-url", "", "Git repository URL for custom tasks")
 	customTaskGitBranch = flag.String("custom-task-git-branch", "", "Git repository branch for custom tasks")
 	customTaskGitCommit = flag.String("custom-task-git-commit", "", "Git commit for custom tasks")
@@ -129,6 +130,7 @@ func main() {
 		Args:                args,
 		CommPort:            *commPort,
 		DownloadAssetsS3:    *downloadAssetsS3,
+		UploadToOCI:         *uploadToOCI,
 		CustomTaskGitURL:    *customTaskGitURL,
 		CustomTaskGitBranch: *customTaskGitBranch,
 		CustomTaskGitCommit: *customTaskGitCommit,

config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml

@@ -276,6 +276,112 @@ spec:
               outputs:
                 description: Outputs specifies storage for evaluation results
                 properties:
+                  oci:
+                    description: Upload results to OCI registry
+                    properties:
+                      dockerConfigJson:
+                        description: DockerConfigJson for registry authentication
+                          (alternative to username/password)
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      password:
+                        description: Password for registry authentication
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      registry:
+                        description: Registry URL (e.g., quay.io, registry.redhat.com)
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      repository:
+                        description: Repository path (e.g., myorg/evaluation-results)
+                        pattern: ^[a-zA-Z0-9._/-]*$
+                        type: string
+                      subject:
+                        description: Subject for the OCI artifact
+                        maxLength: 255
+                        pattern: ^[a-zA-Z0-9._:/@-]*$
+                        type: string
+                      tag:
+                        description: Optional tag for the artifact (defaults to job
+                          name if not specified)
+                        maxLength: 128
+                        pattern: ^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$
+                        type: string
+                      username:
+                        description: Username for registry authentication
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      verifySSL:
+                        description: Whether to verify SSL certificates
+                        type: boolean
+                    required:
+                    - registry
+                    - repository
+                    type: object
                   pvcManaged:
                     description: Create an operator managed PVC
                     properties:

controllers/lmes/driver/driver.go

@@ -67,6 +67,7 @@ type DriverOption struct {
 	Args                []string
 	CommPort            int
 	DownloadAssetsS3    bool
+	UploadToOCI         bool
 	CustomTaskGitURL    string
 	CustomTaskGitBranch string
 	CustomTaskGitCommit string
@@ -242,6 +243,55 @@ func (d *driverImpl) downloadS3Assets() error {
 	return nil
 }
 
+func (d *driverImpl) uploadToOCI() error {
+	if d == nil || !d.Option.UploadToOCI {
+		return nil
+	}
+
+	fmt.Println("Uploading results to OCI registry")
+
+	// Build command arguments: scripts/oci.py <registry> <output_location>
+	registryFromEnv := os.Getenv("OCI_REGISTRY")
+	if registryFromEnv == "" {
+		return fmt.Errorf("OCI_REGISTRY environment variable not set")
+	}
+
+	// Typically the end-user doesn't know how supply the results' subdirectory,
+	// so we hand-over the output path to the script; typically outputPath will contain:
+	// <lmeval subdir>, sterr.log, stout.log, lost+found(directory) and the script
+	// can figure out how to best handle it.
+	outputPath := d.Option.OutputPath
+	scriptPath := "/opt/app-root/src/scripts/oci.py"
+
+	fmt.Printf("[DEBUG] OCI upload CLI: python %s %s %s\n", scriptPath, registryFromEnv, outputPath)
+
+	// List all files and directories in resultsLocation
+	fmt.Printf("[DEBUG] Contents of results location (%s):\n", outputPath)
+	_ = filepath.Walk(outputPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("  [error] %v\n", err)
+			return nil
+		}
+		rel, _ := filepath.Rel(outputPath, path)
+		if rel == "." {
+			fmt.Printf("  %s/\n", rel)
+		} else if info.IsDir() {
+			fmt.Printf("  %s/\n", rel)
+		} else {
+			fmt.Printf("  %s\n", rel)
+		}
+		return nil
+	})
+
+	output, err := exec.Command("python", scriptPath, registryFromEnv, outputPath).Output()
+	fmt.Println(string(output))
+	if err != nil {
+		return fmt.Errorf("failed to upload results to OCI: %v", err)
+	}
+
+	return nil
+}
+
 func patchDevice(args []string, hasCuda bool) []string {
 	device := "cpu"
 	if hasCuda {
@@ -443,6 +493,11 @@ func (d *driverImpl) updateCompleteStatus(err error) {
 		var results string
 		results, err = d.getResults()
 		d.status.Results = results
+
+		// Upload results to OCI if configured
+		if err == nil {
+			err = d.uploadToOCI()
+		}
 	}
 
 	if err != nil {