Add support for v1.30.11 vSphere Kubernetes Release

ansible/files/scripts/docker_registry_ops.py

@@ -0,0 +1,54 @@
+import argparse
+import json
+import subprocess
+
+IMAGE_NAME = "docker.io/vmware/docker-registry"
+PAUSE_IMAGE_NAME = "localhost:5000/vmware.io/pause"
+LABEL = "io.cri-containerd.pinned=pinned"
+
+def get_image_version(image_name):
+    cp = subprocess.run(["crictl", "images", "-o", "json"], capture_output=True, text=True)
+    cp.check_returncode()
+    images = json.loads(cp.stdout)["images"]
+    for image in images:
+        for repo_tag in image["repoTags"]:
+            if repo_tag.startswith(image_name):
+                return repo_tag.split(":")[-1]
+    else:
+        raise Exception(f"No image with name {image_name} found")
+
+def get_registry_version():
+    return get_image_version(IMAGE_NAME)
+
+def get_pause_version():
+    return get_image_version(PAUSE_IMAGE_NAME)
+
+def apply_label(image):
+    subprocess.run(["ctr", "-n", "k8s.io", "images", "label", image, LABEL], check=True)
+
+def pin_image():
+    image_name_with_version = IMAGE_NAME + ":" + get_registry_version()
+    apply_label(image_name_with_version)
+
+    pause_image_name_with_version = PAUSE_IMAGE_NAME + ":" + get_pause_version()
+    apply_label(pause_image_name_with_version)
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Script to copy carvel packages')
+    parser.add_argument('--version',
+                        action='store_true',
+                        help='Print version of docker-registry image')
+    parser.add_argument('--pin',
+                        action='store_true',
+                        help='Pin image by applying label io.cri-containerd.pinned=pinned')
+
+    args = parser.parse_args()
+    if args.version:
+        print(get_registry_version())
+    elif args.pin:
+        pin_image()
+
+
+if __name__ == '__main__':
+    main()

ansible/tasks/main.yml

@@ -1,4 +1,5 @@
-# Copyright 2023 VMware, Inc.
+# © Broadcom. All Rights Reserved.
+# The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
 # SPDX-License-Identifier: MPL-2.0
 ---
 - import_tasks: common.yml
@@ -20,9 +21,11 @@
   register: registry_store_url_check
   ignore_errors: true
 
+# Used in Windows OVA builds
 - import_tasks: registry.yml
   when: registry_store_url_check.status == 200
 
+# Used in Linux OVA builds
 - import_tasks: retag_images.yml
   when: registry_store_url_check.status != 200
 
@@ -35,4 +38,15 @@
   args:
     executable: /bin/bash
 
-- import_tasks: disable_pwd_rotation.yml
+- import_tasks: disable_pwd_rotation.yml
+
+# Gather list of installed packages
+- name: Gather the package facts
+  ansible.builtin.package_facts:
+    manager: auto
+
+- name: Copy installed package details to local file
+  ansible.builtin.copy:
+    content: "{{ ansible_facts.packages | to_nice_json }}"
+    dest: "{{ output_dir }}/package_list.json"
+  delegate_to: localhost

ansible/tasks/retag_images.yml

@@ -1,4 +1,5 @@
-# Copyright 2023 VMware, Inc.
+# (c) Broadcom. All Rights Reserved.
+# The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
 # SPDX-License-Identifier: MPL-2.0
 ---
 - name: Retag Container Images
@@ -14,6 +15,11 @@
   args:
     executable: python3
 
+- name: Pin container images
+  ansible.builtin.script: files/scripts/docker_registry_ops.py --pin
+  args:
+    executable: python3
+
 - name: Stop docker registry
   shell: 'ctr -n k8s.io task kill docker-registry'
 

build-ova.sh

@@ -31,6 +31,10 @@ function download_configuration_files() {
     # Download compatibility files
     wget -q http://${HOST_IP}:${ARTIFACTS_CONTAINER_PORT}/artifacts/metadata/compatibility/vmware-system.compatibilityoffering.json
     wget -q http://${HOST_IP}:${ARTIFACTS_CONTAINER_PORT}/artifacts/metadata/compatibility/vmware-system.guest.kubernetes.distribution.image.version.json
+
+    # Download VKr constraints files
+    wget -q http://${HOST_IP}:${ARTIFACTS_CONTAINER_PORT}/artifacts/metadata/vmware-system.kr.destination-semver-constraint.json || echo "override-semver-constraint.json don't exist"
+    wget -q http://${HOST_IP}:${ARTIFACTS_CONTAINER_PORT}/artifacts/metadata/vmware-system.kr.override-semver-constraint.json || echo "override-semver-constraint.json don't exist"
 }
 
 # Generate packaer input variables based on packer-variables folder

goss/goss-command.yaml

@@ -1,4 +1,5 @@
-# Copyright 2023 VMware, Inc.
+# © Broadcom. All Rights Reserved.
+# The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
 # SPDX-License-Identifier: MPL-2.0
 
 command:
@@ -10,9 +11,15 @@ command:
     timeout: 0
   crictl ps:
     exit-status: 0
-    stdout: []
-    stderr: []
     timeout: 0
+  ctr -n k8s.io image ls 2>/dev/null | grep docker-registry | grep 'io.cri-containerd.pinned=pinned':
+    exit-status: 0
+    stderr: []
+    timeout: 10000 # in ms
+  ctr -n k8s.io image ls 2>/dev/null | grep pause | grep 'io.cri-containerd.pinned=pinned':
+    exit-status: 0
+    stderr: []
+    timeout: 10000 # in ms
 {{if ne .Vars.containerd_wasm_shims_runtimes ""}}
   containerd-shim-lunatic-v1:
     exit-status: 1

packer-variables/default-args.j2

@@ -38,5 +38,6 @@
     "remove_extra_repos": "true",
     {# Packer VM name #}
     "build_version": "{{ os_type }}-kube-{{ kubernetes_series }}-{{ ova_ts_suffix }}",
-    "custom_role": "true"
+    "custom_role": "true",
+    "output_dir": "/image-builder/images/capi/output/{{ os_type }}-kube-{{ kubernetes_series }}-{{ ova_ts_suffix }}"
 }

packer-variables/photon-5/default-args-photon-5.j2

@@ -3,8 +3,8 @@
     location after mounting that Path using docker #}
     "custom_role_names": "/image-builder/images/capi/image/ansible /image-builder/images/capi/image/compliance",
     "distro_version": "5.0",
-    "extra_rpms": "glibc zlib filesystem ethtool pkg-config bash bzip2 shadow procps-ng iana-etc coreutils bc libtool findutils xz iproute2 util-linux kmod linux linux-devel iptables Linux-PAM systemd dbus file e2fsprogs rpm gawk cloud-utils gptfdisk nfs-utils openssh gdbm photon-release photon-repos haveged sed grep cpio gzip vim tdnf less iputils bindutils diffutils bridge-utils cri-tools apparmor-utils apparmor-profiles krb5 which tzdata motd sudo iotop lsof traceroute ethtool dstat ltrace ipset netcat tcpdump wget net-tools curl tar open-vm-tools libseccomp cloud-init sysstat jq conntrack dkms",
+    "extra_rpms": "glibc zlib filesystem ethtool pkg-config bash bzip2 shadow procps-ng iana-etc coreutils bc libtool findutils xz iproute2 util-linux kmod linux linux-devel iptables Linux-PAM systemd dbus file e2fsprogs rpm gawk cloud-utils gptfdisk nfs-utils openssh gdbm photon-release photon-repos haveged sed grep cpio gzip vim tdnf less iputils bindutils diffutils bridge-utils cri-tools apparmor-utils apparmor-profiles krb5 which tzdata motd sudo iotop lsof traceroute ethtool dstat ltrace ipset netcat tcpdump wget net-tools curl tar open-vm-tools libseccomp cloud-init sysstat jq conntrack dkms python3-rpm",
     {# additional variables passed to the ansible role, if stig is enabled #}
     {# variables passed to the ansible role are appended to this list at the end #}
-    "ansible_user_vars": "artifacts_container_url=http://{{ host_ip }}:{{ artifacts_container_port }} dockerVersion={{ docker_distribution }} imageVersion={{ image_version|replace('-', '.') }} ansible_python_interpreter=/usr/bin/python3 addon_image_list=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/calico.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/metrics-server.tar{% if capabilities_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/capabilities.tar{% endif %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/guest-cluster-auth-service.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/pinniped.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/antrea.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-cpi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-pv-csi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/secretgen-controller.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/kapp-controller.tar{% if gateway_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/gateway-api.tar{% endif %} localhost_addon_image_list={{ calico_package_localhost_path }},{{ metrics_server_package_localhost_path }}{% if capabilities_package_present %},{{ capabilities_package_localhost_path }}{% endif %},{{ guest_cluster_auth_service_package_localhost_path }},{{ pinniped_package_localhost_path }},{{ antrea_package_localhost_path }},{{ vsphere_cpi_package_localhost_path }},{{ vsphere_pv_csi_package_localhost_path }},{{ secretgen_controller_package_localhost_path }},{{ kapp_controller_localhost_path }}{% if gateway_package_present %},{{ gateway_api_package_localhost_path }}{% endif %} networkd_dispatcher_download_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/networkd-dispatcher-2.1.tar.bz2 registry_store_archive_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/registries/{{ registry_store_path }} run_sysctl_net_ipv6_conf_all_mc_forwarding=false run_sysctl_net_ipv6_conf_default_mc_forwarding=false run_sysctl_net_ipv6_conf_eth0_mc_forwarding=false run_kernel_net_ip_forward=false run_modprobe_conf=false run_fips_boot_enable=false run_openssl_fips=false run_sshd_banner=false"
+    "ansible_user_vars": "artifacts_container_url=http://{{ host_ip }}:{{ artifacts_container_port }} dockerVersion={{ docker_distribution }} imageVersion={{ image_version|replace('-', '.') }} ansible_python_interpreter=/usr/bin/python3 addon_image_list=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/calico.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/metrics-server.tar{% if capabilities_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/capabilities.tar{% endif %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/guest-cluster-auth-service.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/pinniped.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/antrea.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-cpi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-pv-csi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/secretgen-controller.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/kapp-controller.tar{% if gateway_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/gateway-api.tar{% endif %} localhost_addon_image_list={{ calico_package_localhost_path }},{{ metrics_server_package_localhost_path }}{% if capabilities_package_present %},{{ capabilities_package_localhost_path }}{% endif %},{{ guest_cluster_auth_service_package_localhost_path }},{{ pinniped_package_localhost_path }},{{ antrea_package_localhost_path }},{{ vsphere_cpi_package_localhost_path }},{{ vsphere_pv_csi_package_localhost_path }},{{ secretgen_controller_package_localhost_path }},{{ kapp_controller_localhost_path }}{% if gateway_package_present %},{{ gateway_api_package_localhost_path }}{% endif %} networkd_dispatcher_download_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/networkd-dispatcher-2.1.tar.bz2 registry_store_archive_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/registries/{{ registry_store_path }} run_sysctl_net_ipv6_conf_all_mc_forwarding=false run_sysctl_net_ipv6_conf_default_mc_forwarding=false run_sysctl_net_ipv6_conf_eth0_mc_forwarding=false run_kernel_net_ip_forward=false run_modprobe_conf=false run_fips_boot_enable=false run_openssl_fips=false run_sshd_banner=false output_dir=/image-builder/images/capi/output/{{ os_type }}-kube-{{ kubernetes_series }}-{{ ova_ts_suffix }}"
 }

packer-variables/ubuntu-2204-efi/default-args-ubuntu-2204-efi.j2

@@ -2,6 +2,7 @@
     "custom_role_names": "/image-builder/images/capi/image/ansible",
     "distro_version": "22.04",
     "extra_debs": "unzip iptables-persistent nfs-common dkms",
+    "ansible_user_vars": "artifacts_container_url=http://{{ host_ip }}:{{ artifacts_container_port }} dockerVersion={{ docker_distribution }} imageVersion={{ image_version|replace('-', '.') }} ansible_python_interpreter=/usr/bin/python3 addon_image_list=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/calico.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/metrics-server.tar{% if capabilities_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/capabilities.tar{% endif %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/guest-cluster-auth-service.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/pinniped.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/antrea.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-cpi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/vsphere-pv-csi.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/secretgen-controller.tar,http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/kapp-controller.tar{% if gateway_package_present %},http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/packages/gateway-api.tar{% endif %} localhost_addon_image_list={{ calico_package_localhost_path }},{{ metrics_server_package_localhost_path }}{% if capabilities_package_present %},{{ capabilities_package_localhost_path }}{% endif %},{{ guest_cluster_auth_service_package_localhost_path }},{{ pinniped_package_localhost_path }},{{ antrea_package_localhost_path }},{{ vsphere_cpi_package_localhost_path }},{{ vsphere_pv_csi_package_localhost_path }},{{ secretgen_controller_package_localhost_path }},{{ kapp_controller_localhost_path }}{% if gateway_package_present %},{{ gateway_api_package_localhost_path }}{% endif %} networkd_dispatcher_download_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/networkd-dispatcher-2.1.tar.bz2 registry_store_archive_url=http://{{ host_ip }}:{{ artifacts_container_port }}/artifacts/{{ kubernetes_version }}/registries/{{ registry_store_path }} output_dir=/image-builder/images/capi/output/{{ os_type }}-kube-{{ kubernetes_series }}-{{ ova_ts_suffix }}",
     "boot_disable_ipv6": "1",
     "ip_settle_timeout": "15m"
 }

scripts/tkg_byoi.py

@@ -232,6 +232,13 @@ def copy_ova(args):
     old_path = os.path.join(default_ova_destination_folder, old_ova_name)
     print("Copying OVA from {} to {}".format(old_path, new_path))
     shutil.copyfile(old_path, new_path)
+
+    # Copy the package list
+    old_path = os.path.join(default_ova_destination_folder, "package_list.json")
+    new_path = os.path.join(args.ova_destination_folder, "package_list.json")
+    print("Copying package list file from {} to {}".format(old_path, new_path))
+    shutil.copyfile(old_path, new_path)
+
     print("Copying completed")
 
 

scripts/utkg_custom_ovf_properties.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-# Copyright 2023 VMware, Inc.
+# © Broadcom. All Rights Reserved.
+# The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
 # SPDX-License-Identifier: MPL-2.0
 
 import argparse
@@ -93,6 +94,17 @@ def create_non_addon_ovf_properties():
             key = Path(file).stem
             custom_ovf_properties[key] = convert_to_xml(data)
 
+def create_non_addon_VKr_constraints_ovf_properties():
+    filenames = [ join(tkg_core_directory,"vmware-system.kr.destination-semver-constraint.json"),
+                 join(tkg_core_directory,"vmware-system.kr.override-k8s-semver-version.json")]
+    for file in filenames:
+        try:
+            with open(file) as f:
+                data = json.dumps(json.load(f)).replace('"','')
+                key = Path(file).stem
+                custom_ovf_properties[key] = convert_to_xml(compress_and_base64_encode(data))
+        except IOError:
+            print("couldn't find/read file: ",file)   
 
 # fetch tkr apiversion and tkr version
 def fetch_tkr_data():
@@ -328,6 +340,7 @@ def main():
     set_versions(args)
     create_utkg_tkr_metadata_ovf_properties()
     create_non_addon_ovf_properties()
+    create_non_addon_VKr_constraints_ovf_properties()
     write_properties_to_file(args.outfile)
     print(custom_ovf_properties)
 

supported-context.json

@@ -4,7 +4,7 @@
         "ubuntu-2204-efi",
         "windows-2022-efi"
     ],
-    "artifacts_image": "projects.packages.broadcom.com/vsphere/iaas/kubernetes-release/1.30.8/tkg-vsphere-linux-resource-bundle:v1.30.8_vmware.1-fips-vkr.1",
+    "artifacts_image": "projects.packages.broadcom.com/vsphere/iaas/kubernetes-release/1.30.11/tkg-vsphere-linux-resource-bundle:v1.30.11_vmware.1-fips-vkr.2",
     "docker_build_args": {
         "IMAGE_BUILDER_COMMIT_ID": "49377913fe89429542805fd809ca1eabd5a93743"
     }

supported-version.txt

@@ -1 +1 @@
-v1.30.8+vmware.1-fips
+v1.30.11+vmware.1-fips