Skip to content

IB Support for Slurm and K8s #3826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jagadeeshnv merged 3 commits into from
Jan 5, 2026
Merged

IB Support for Slurm and K8s #3826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bc94810
modify templates for doca ofed
VrindaMarwah Dec 31, 2025
2766c97
doca ofed installation changes for k8s
VrindaMarwah Dec 31, 2025
a71819d
add ansible builtin
VrindaMarwah Jan 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .../roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '0755'
content: |
Expand Down Expand Up @@ -169,6 +175,7 @@
{{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}

runcmd:
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/set-ssh.sh
- /usr/local/bin/install_cuda_toolkit.sh
- groupadd -r {{ slurm_group_name }}
Expand Down
7 changes: 7 additions & 0 deletions ...y/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '0755'
content: |
Expand Down Expand Up @@ -171,6 +177,7 @@
{{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}

runcmd:
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/set-ssh.sh
- /usr/local/bin/install_cuda_toolkit.sh
- groupadd -r {{ slurm_group_name }}
Expand Down
7 changes: 7 additions & 0 deletions discovery/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -92,6 +98,7 @@
{{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}

runcmd:
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/set-ssh.sh
- groupadd -r {{ slurm_group_name }}
- useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}
Expand Down
7 changes: 7 additions & 0 deletions discovery/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -91,6 +97,7 @@
{{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}

runcmd:
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/set-ssh.sh
- groupadd -r {{ slurm_group_name }}
- useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}
Expand Down
16 changes: 12 additions & 4 deletions ...gure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '0755'
content: |
Expand Down Expand Up @@ -90,6 +96,7 @@
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0
tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0
permissions: '0644'

Expand Down Expand Up @@ -369,7 +376,7 @@
- sudo modprobe nf_conntrack || true
- sudo modprobe vxlan || true
- sysctl --system
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd /var/lib/kubelet /etc/kubernetes /var/log/pods
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
- |
tmpfile=$(mktemp)
# Extract the first 'search' line only (ignore duplicates)
Expand All @@ -391,14 +398,15 @@
chattr +i /etc/resolv.conf || true
fi
- mount -a
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh
- systemctl start crio.service
- systemctl enable crio.service
- sudo systemctl enable --now kubelet
- mv /tmp/crio.conf /etc/containers/registries.conf.d/crio.conf
- mv /tmp/generate-control-plane-join.sh {{ k8s_client_mount_path }}
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- systemctl daemon-reload
- systemctl restart crio
- kubeadm config images pull --kubernetes-version={{ service_k8s_version }}
Expand Down
16 changes: 12 additions & 4 deletions .../configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '0755'
content: |
Expand Down Expand Up @@ -77,6 +83,7 @@
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0
tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0
permissions: '0644'
- path: /etc/containers/storage.conf
Expand Down Expand Up @@ -276,7 +283,7 @@
- sudo modprobe nf_conntrack || true
- sudo modprobe vxlan || true
- sysctl --system
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd /var/lib/kubelet /etc/kubernetes /var/log/pods
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
- |
tmpfile=$(mktemp)

Expand All @@ -299,15 +306,16 @@
chattr +i /etc/resolv.conf || true
fi
- mount -a
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh
- systemctl start crio.service
- systemctl enable crio.service
- sudo systemctl enable --now kubelet
- mv /tmp/crio.conf /etc/containers/registries.conf.d/crio.conf
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- systemctl daemon-reload
- systemctl restart crio
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- kubeadm config images pull --kubernetes-version={{ service_k8s_version }}
- echo "Installing helm"
- /usr/local/bin/install-helm.sh
Expand Down
16 changes: 12 additions & 4 deletions ...ery/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_node_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '0755'
content: |
Expand Down Expand Up @@ -76,6 +82,7 @@
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0
{{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0
tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0
permissions: '0644'
- path: /etc/containers/storage.conf
Expand Down Expand Up @@ -178,7 +185,7 @@
- sudo modprobe nf_conntrack || true
- sudo modprobe vxlan || true
- sysctl --system
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/kubelet /etc/kubernetes /var/log/pods
- mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
- |
tmpfile=$(mktemp)

Expand All @@ -202,14 +209,15 @@
fi
- systemctl restart rpcbind
- mount -a
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh
- systemctl start crio.service
- systemctl enable crio.service
- sudo systemctl enable --now kubelet
- mv /tmp/crio.conf /etc/containers/registries.conf.d/crio.conf
- cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors
- update-ca-trust extract
- systemctl daemon-reload
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- systemctl restart crio
- kubeadm config images pull --kubernetes-version={{ service_k8s_version }}
- |
Expand Down
15 changes: 11 additions & 4 deletions ...ry/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -243,7 +249,7 @@
- useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}

# Create directories for nfs and mount all
- mkdir -p /var/log/slurm /etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track
- mkdir -p /var/log/slurm /etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track /var/lib/packages
- echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab
- echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm /etc/slurm nfs defaults,_netdev 0 0" >> /etc/fstab
- echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/my.cnf.d /etc/my.cnf.d nfs defaults,_netdev 0 0" >> /etc/fstab
Expand All @@ -253,8 +259,12 @@
- echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool /var/spool nfs defaults,_netdev 0 0" >> /etc/fstab
- echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab
- echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab
- echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab
- chmod {{ file_mode }} /etc/fstab
- mount -a
- cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh

- chown -R {{ user }}:{{ slurm_group_name }} {{ home_dir }}
- chmod {{ file_mode_755 }} {{ home_dir }}
Expand All @@ -277,9 +287,6 @@
- systemctl enable sshd
- systemctl start sshd

- cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf

{% if hostvars['localhost']['openldap_support'] %}
- /usr/local/bin/update_ldap_conf.sh
- mkdir /ldapcerts
Expand Down
14 changes: 13 additions & 1 deletion discovery/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -220,7 +226,7 @@
echo "[INFO] ===== Starting directory creation and NFS mounts for Slurm and Munge (aarch64) ====="

echo "[INFO] Creating base directories for Slurm and Munge"
mkdir -pv /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /var/log/track
mkdir -pv /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /var/log/track /var/lib/packages

echo "[INFO] Updating /etc/fstab with NFS entries for Slurm and Munge paths"
echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm /var/log/slurm nfs defaults,_netdev 0 0" >> /etc/fstab
Expand All @@ -229,6 +235,9 @@
echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool /var/spool nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab

chmod {{ file_mode }} /etc/fstab

echo "[INFO] Mounting all NFS entries from /etc/fstab"
Expand Down Expand Up @@ -376,6 +385,9 @@
- useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}

- /usr/local/bin/configure_dirs_and_mounts.sh
- cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/configure_slurmd_setup.sh
- /usr/local/bin/configure_munge_and_pam.sh

Expand Down
15 changes: 11 additions & 4 deletions discovery/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
disable_root: false

write_files:
- path: /usr/local/bin/doca-install.sh
owner: root:root
permissions: '{{ file_mode_755 }}'
content: |
{{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

- path: /usr/local/bin/set-ssh.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -229,7 +235,7 @@
echo "[INFO] ===== Starting directory creation and NFS mounts for Pulp cert, Slurm and Munge ====="

echo "[INFO] Creating base directories for Pulp cert, Slurm and Munge"
mkdir -pv /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /cert /var/log/track
mkdir -pv /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /cert /var/log/track /var/lib/packages

echo "[INFO] Updating /etc/fstab with NFS entries for Pulp cert, Slurm and Munge paths"
echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab
Expand All @@ -238,6 +244,7 @@
echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab
echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab
chmod {{ file_mode }} /etc/fstab

echo "[INFO] Mounting all NFS entries from /etc/fstab"
Expand Down Expand Up @@ -379,11 +386,11 @@
- useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}

- /usr/local/bin/configure_dirs_and_mounts.sh
- /usr/local/bin/configure_slurmd_setup.sh
- /usr/local/bin/configure_munge_and_pam.sh

- cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
- sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
- bash /usr/local/bin/doca-install.sh
- /usr/local/bin/configure_slurmd_setup.sh
- /usr/local/bin/configure_munge_and_pam.sh

- setenforce 0
- /usr/local/bin/configure_firewall_and_services.sh
Expand Down
Loading