Allow to prune operators from different catalogs

betoredhat · betoredhat · commit 9c1985f7885a · 2025-11-25T17:40:07.000-06:00
diff --git a/roles/imageset_mirroring/README.md b/roles/imageset_mirroring/README.md
@@ -32,10 +32,10 @@ This role provides three different approaches to mirror operators from a source
 
 #### mirror-latest & mirror-all
 
-| Variable          | Type       | Required | Description                                                                               |
-|-------------------|------------|----------|-------------------------------------------------------------------------------------------|
-| `im_source_index` | string     | yes      | Source catalog index (e.g., `registry.redhat.io/redhat/redhat-operator-index:v4.20`)      |
-| `im_operators`    | dictionary | yes      | Dictionary of operators to mirror (keys are operator names)                               |
+| Variable          | Type       | Required | Description                                                                                                                                                          |
+|-------------------|------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `im_source_index` | string     | yes      | Default source catalog index (e.g., `registry.redhat.io/redhat/redhat-operator-index:v4.20`). Can be overridden per-operator.                                       |
+| `im_operators`    | dictionary | yes      | Dictionary of operators to mirror (keys are operator names). Each operator value can be an empty dict or contain a `catalog` key to override the default catalog.  |
 
 #### mirror-custom
 
@@ -63,6 +63,29 @@ This role provides three different approaches to mirror operators from a source
     im_auths_file: /path/to/pull-secret.json
 ```
 
+### Example 1c: Mirror Latest Version from Multiple Catalogs
+
+```yaml
+- name: "Mirror operators from different catalogs"
+  ansible.builtin.include_role:
+    name: redhatci.ocp.imageset_mirroring
+  vars:
+    im_task: latest
+    im_source_index: registry.redhat.io/redhat/redhat-operator-index:v4.20
+    im_target: registry.lab:4443
+    im_operators:
+      # These operators use the default im_source_index
+      mcg-operator:
+      ocs-client-operator:
+      # This operator overrides the catalog
+      custom-operator:
+        catalog: quay.io/prega/prega-operator-index:v4.21-latest
+      another-custom-operator:
+        catalog: quay.io/prega/prega-operator-index:v4.21-latest
+    im_allow_insecure_registries: true
+    im_auths_file: /path/to/pull-secret.json
+```
+
 ### Example 1b: Mirror Latest Version (using tasks_from - legacy)
 
 ```yaml
@@ -96,6 +119,23 @@ This role provides three different approaches to mirror operators from a source
       rhacm-operator:
 ```
 
+### Example 2b: Mirror All Versions from Multiple Catalogs
+
+```yaml
+- name: "Mirror all operators from multiple catalogs"
+  ansible.builtin.include_role:
+    name: redhatci.ocp.imageset_mirroring
+  vars:
+    im_task: all
+    im_source_index: registry.redhat.io/redhat/redhat-operator-index:v4.20
+    im_target: registry.lab:4443
+    im_operators:
+      openshift-gitops-operator:
+      rhacm-operator:
+      prega-operator:
+        catalog: quay.io/prega/prega-operator-index:v4.21-20251111T044345
+```
+
 ### Example 3: Mirror Using Custom ImageSetConfiguration
 
 ```yaml
@@ -167,16 +207,18 @@ This role provides three different approaches to mirror operators from a source
 ### mirror-latest
 
 1. Downloads the latest `oc-mirror` binary
-2. Queries the source catalog for each operator's default channel
-3. Identifies the latest CSV version for the default channel
-4. Generates an ImageSetConfiguration with specific version constraints
-5. Executes `oc-mirror` to perform the mirroring
+2. Groups operators by their catalog source (using `im_source_index` as default)
+3. Queries each catalog for each operator's default channel
+4. Identifies the latest CSV version for the default channel
+5. Generates an ImageSetConfiguration with multiple catalog entries (if operators use different catalogs)
+6. Executes `oc-mirror` to perform the mirroring
 
 ### mirror-all
 
 1. Downloads the latest `oc-mirror` binary
-2. Generates an ImageSetConfiguration with `full: true` for all operators
-3. Executes `oc-mirror` to mirror all channels and versions
+2. Groups operators by their catalog source (using `im_source_index` as default)
+3. Generates an ImageSetConfiguration with `full: true` for all operators, organized by catalog
+4. Executes `oc-mirror` to mirror all channels and versions
 
 ### mirror-custom
 
diff --git a/roles/imageset_mirroring/meta/argument_specs.yml b/roles/imageset_mirroring/meta/argument_specs.yml
@@ -35,11 +35,16 @@ argument_specs:
       im_source_index:
         type: str
         required: true
-        description: Source catalog index (e.g., registry.redhat.io/redhat/redhat-operator-index:v4.20).
+        description: |
+          Default source catalog index (e.g., registry.redhat.io/redhat/redhat-operator-index:v4.20).
+          Can be overridden per-operator in im_operators dictionary.
       im_operators:
         type: dict
         required: true
-        description: Dictionary of operators to mirror (keys are operator names).
+        description: |
+          Dictionary of operators to mirror (keys are operator names).
+          Each operator can optionally specify a 'catalog' key to override im_source_index.
+          Example: {'operator1': {}, 'operator2': {'catalog': 'custom.io/index:v1.0'}}
       im_allow_insecure_registries:
         type: bool
         required: false
@@ -62,11 +67,16 @@ argument_specs:
       im_source_index:
         type: str
         required: true
-        description: Source catalog index (e.g., registry.redhat.io/redhat/redhat-operator-index:v4.20).
+        description: |
+          Default source catalog index (e.g., registry.redhat.io/redhat/redhat-operator-index:v4.20).
+          Can be overridden per-operator in im_operators dictionary.
       im_operators:
         type: dict
         required: true
-        description: Dictionary of operators to mirror (keys are operator names).
+        description: |
+          Dictionary of operators to mirror (keys are operator names).
+          Each operator can optionally specify a 'catalog' key to override im_source_index.
+          Example: {'operator1': {}, 'operator2': {'catalog': 'custom.io/index:v1.0'}}
       im_allow_insecure_registries:
         type: bool
         required: false
diff --git a/roles/imageset_mirroring/tasks/mirror-all.yml b/roles/imageset_mirroring/tasks/mirror-all.yml
@@ -6,6 +6,25 @@
       - im_operators | dict2items | length > 0
     fail_msg: "im_operators must contain at least one operator"
 
+- name: Build operators with catalog mapping
+  ansible.builtin.set_fact:
+    _im_operators_with_catalog: >-
+      {{
+        _im_operators_with_catalog | default([]) +
+        [{
+          'catalog': item.value.catalog | default(im_source_index),
+          'operator': item.key
+        }]
+      }}
+  loop: "{{ im_operators | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Group operators by catalog for template
+  ansible.builtin.set_fact:
+    _im_catalogs_with_operators: >-
+      {{ _im_operators_with_catalog | groupby('catalog') }}
+
 - name: Render ImageSetConfiguration for all versions of listed operators
   ansible.builtin.template:
     src: imagesetconfig-full.yaml.j2
diff --git a/roles/imageset_mirroring/tasks/mirror-latest.yml b/roles/imageset_mirroring/tasks/mirror-latest.yml
@@ -11,7 +11,7 @@
     cmd: |
       set -o pipefail
       {{ _im_tmp_dir.path }}/oc-mirror list operators \
-        --catalog {{ im_source_index }} \
+        --catalog {{ item.value.catalog | default(im_source_index) }} \
         --package {{ item.key }} \
         2>/dev/null | \
       awk 'NR==2 {operator=$1; def_channel=$NF}
@@ -24,20 +24,31 @@
   retries: 3
   delay: 10
   loop: "{{ im_operators | dict2items }}"
+  loop_control:
+    label: "{{ item.key }} from {{ item.value.catalog | default(im_source_index) }}"
   changed_when: false
 
+- name: Build operators with catalog mapping
+  ansible.builtin.set_fact:
+    _im_operators_with_catalog: >-
+      {{
+        _im_operators_with_catalog | default([]) +
+        [{
+          'catalog': item.item.value.catalog | default(im_source_index),
+          'stdout': item.stdout,
+          'operator': item.item.key
+        }]
+      }}
+  loop: "{{ _im_operator_info.results }}"
+  loop_control:
+    label: "{{ item.item.key }}"
+
+- name: Group operators by catalog for template
+  ansible.builtin.set_fact:
+    _im_catalogs_with_operators: >-
+      {{ _im_operators_with_catalog | groupby('catalog') }}
+
 - name: Render ImageSetConfiguration for latest version of listed operators
-  vars:
-    im_packages_var: |
-      {% for result in _im_operator_info.results %}
-      {%- set parts = result.stdout.split() %}
-      {%- set version = parts[2].split('.v')[-1] if '.v' in parts[2] else parts[2].split('-')[-1] %}
-      - name: '{{ parts[0] }}'
-        channels:
-          - name: '{{ parts[1] }}'
-            minVersion: '{{ version }}'
-            maxVersion: '{{ version }}'
-      {% endfor %}
   ansible.builtin.template:
     src: imagesetconfig-latest.yaml.j2
     dest: "{{ _im_tmp_dir.path }}/imagesetconfig.yaml"
diff --git a/roles/imageset_mirroring/templates/imagesetconfig-full.yaml.j2 b/roles/imageset_mirroring/templates/imagesetconfig-full.yaml.j2
@@ -2,9 +2,11 @@ apiVersion: mirror.openshift.io/v2alpha1
 kind: ImageSetConfiguration
 mirror:
   operators:
-    - catalog: {{ im_source_index }}
+{% for catalog, operators in _im_catalogs_with_operators %}
+    - catalog: {{ catalog }}
       full: true
       packages:
-{% for package in im_operators.keys() %}
-        - name: {{ package }}
+{% for op in operators %}
+        - name: {{ op.operator }}
+{% endfor %}
 {% endfor %}
diff --git a/roles/imageset_mirroring/templates/imagesetconfig-latest.yaml.j2 b/roles/imageset_mirroring/templates/imagesetconfig-latest.yaml.j2
@@ -2,7 +2,26 @@ apiVersion: mirror.openshift.io/v2alpha1
 kind: ImageSetConfiguration
 mirror:
   operators:
-    - catalog: {{ im_source_index }}
+{% for catalog, results in _im_catalogs_with_operators %}
+    - catalog: {{ catalog }}
       full: false
       packages:
-        {{ im_packages_var | indent( width=8, first=false)}}
+{% for result in results %}
+{%- set parts = result.stdout.split() %}
+{%- set csv_full = parts[2] %}
+{%- if '.v' in csv_full %}
+{%-   set version = csv_full.split('.v')[-1] %}
+{%- elif csv_full.startswith(parts[0] + '.') %}
+{%-   set version = csv_full[parts[0]|length + 1:] %}
+{%- elif csv_full.startswith(parts[0] + '-') %}
+{%-   set version = csv_full[parts[0]|length + 1:] %}
+{%- else %}
+{%-   set version = csv_full.split('-')[-1] %}
+{%- endif %}
+        - name: '{{ parts[0] }}'
+          channels:
+            - name: '{{ parts[1] }}'
+              minVersion: '{{ version }}'
+              maxVersion: '{{ version }}'
+{% endfor %}
+{% endfor %}
diff --git a/roles/prune_catalog/README.md b/roles/prune_catalog/README.md
@@ -106,6 +106,35 @@ Another example of a pruned catalog with specific channels
 ```
 
 In this example, the resulting catalog contains all the channels of the `compliance-operator`, channel `stable-5.8` for cluster-logging and only the `stable` channel for `file-integrity-operator`.
+
+### Prune operators from multiple source catalogs
+
+The role supports pruning operators from multiple source catalogs into a single destination catalog. This is useful when you need operators from different catalog sources (e.g., Red Hat operators, certified operators, community operators).
+
+```yaml
+- name: "Create a pruned catalog with operators from multiple catalogs"
+  include_role:
+    name: redhatci.ocp.prune_catalog
+  vars:
+    pc_source_catalog: "registry.redhat.io/redhat/redhat-operator-index:v4.19"
+    pc_destination_catalog: "<my-local-registry>:4443/pruned-catalog:latest"
+    pc_operators:
+      compliance-operator:
+        channel: stable
+      file-integrity-operator:
+        channel: stable
+        catalog: "registry.redhat.io/redhat/certified-operator-index:v4.19"
+      ocs-operator:
+        channel: stable-4.13
+        catalog: "registry.redhat.io/redhat/redhat-operator-index:v4.19"
+      prometheus:
+        channel: beta
+        catalog: "registry.redhat.io/redhat/community-operator-index:v4.19"
+    pc_opm_args: "--skip-tls-verify=true"
+```
+
+- Operators without a `catalog` field use the `pc_source_catalog`
+
 ```ShellSession
 $ DOCKER_CONFIG=/home/<user>/.docker oc-mirror list operators --catalog <my-local-registry>:4443/pruned-catalog:latest
 NAME                     DISPLAY NAME               DEFAULT CHANNEL
diff --git a/roles/prune_catalog/tasks/main.yml b/roles/prune_catalog/tasks/main.yml
@@ -81,6 +81,37 @@
       retries: 3
       delay: 10
 
+    - name: "Render additional catalogs"
+      vars:
+        extra_catalogs: >-
+          {{
+            (pc_operators | type_debug == "dict") |
+            ternary(
+              pc_operators | dict2items |
+              selectattr('value.catalog', 'defined') |
+              map(attribute='value.catalog') |
+              list | unique |
+              difference([pc_source_catalog]),
+              []
+            )
+          }}
+      ansible.builtin.shell:
+        chdir: "{{ pc_tmp_dir }}"
+        cmd: >
+          set -x;
+          {{ pc_opm_cmd }} render {{ catalog }} >>
+          index-packages
+      loop: "{{ extra_catalogs }}"
+      loop_control:
+        loop_var: catalog
+      register: _pc_render_additional
+      changed_when: _pc_render_additional.rc == 0
+      until: _pc_render_additional.rc == 0
+      retries: 3
+      delay: 10
+      when:
+        - extra_catalogs | length > 0
+
     - name: "Extract the listed channels for each operator"
       when: pc_operators | type_debug == "dict"
       block:
