Merge pub/k8s_telemetry to staging

common/library/module_utils/input_validation/schema/omnia_config.json

@@ -86,12 +86,18 @@
             "type": "string",
             "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$"
 
+          },
+          "k8s_crio_storage_size": {
+            "description": "Storage size for CRI-O in Gigabytes only (example: 10G, 15G, 100G)",
+            "type": "string",
+            "pattern": "^[1-9][0-9]*G$"
           }
         },
         "required": [
           "cluster_name",
           "k8s_cni",
-          "k8s_service_addresses"
+          "k8s_service_addresses",
+          "k8s_crio_storage_size"
         ],
         "allOf": [
         {

common/library/module_utils/input_validation/schema/storage_config.json

@@ -31,11 +31,11 @@
             },
             "server_share_path": {
               "type": "string",
-              "pattern": "^/(?:[^/]+(?:/[^/]+)*)?$"
+              "pattern": "^/(?:[^/]+(?:/[^/]+)*)?/?$"
             },
             "client_share_path": {
               "type": "string",
-              "pattern": "^/(?:[^/]+(?:/[^/]+)*)?$"
+              "pattern": "^/(?:[^/]+(?:/[^/]+)*)?/?$"
             },
             "client_mount_options": {
               "type": "string"
@@ -54,4 +54,4 @@
     "required": [
       "nfs_client_params"
     ]
-  }
+  }

common/library/modules/enable_telemetry_service.py

@@ -0,0 +1,211 @@
+# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Dell iDRAC Telemetry - FAST Enable All Reports.
+
+Optimized with parallel processing and connection pooling.
+Supports iDRAC 9 and iDRAC 10.
+"""
+
+import logging
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict, List, Optional, Any, Tuple
+import requests
+import urllib3
+from ansible.module_utils.basic import AnsibleModule
+
+# Disable SSL warnings
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+#####################################################
+# ALL 37 TELEMETRY REPORTS (iDRAC 9 & 10)
+#####################################################
+
+ALL_REPORTS: List[str] = [
+    "AggregationMetrics", "CPUMemMetrics", "CPURegisters",
+    "CPUSensor", "MemoryMetrics", "MemorySensor",
+    "NVMeSMARTData", "StorageDiskSMARTData", "StorageSensor",
+    "NICSensor", "NICStatistics", "FCPortStatistics",
+    "FCSensor", "SFPTransceiver", "InfiniBandStatistics",
+    "PSUMetrics", "PowerMetrics", "PowerStatistics",
+    "FanSensor", "ThermalMetrics", "ThermalSensor",
+    "GPUMetrics", "GPUStatistics", "GPUSubsystemPower", "FPGASensor",
+    "Sensor", "SerialLog", "SystemUsage", "x86SubsystemPower",
+    "OME-ISM-MetricsData", "OME-PMP-Power-B",
+    "OME-SFPTransceiver-Metrics", "OME-Telemetry-FCPortStatistics",
+    "OME-Telemetry-GPU-Aggregate", "OME-Telemetry-GPU-Aggregate-1",
+    "OME-Telemetry-NIC-Statistics", "OME-Telemetry-SMARTData",
+]
+
+def get_report_definitions(
+    ip_address: str,
+    user: str,
+    password: str,
+    session: requests.Session,
+    timeout: int,
+) -> Optional[List[str]]:
+    """Fetch available report definitions from iDRAC."""
+    url = f"https://{ip_address}/redfish/v1/TelemetryService/MetricReportDefinitions"
+    try:
+        response = session.get(
+            url,
+            auth=(user, password),
+            verify=False,
+            timeout=timeout,
+        )
+        if response.status_code == 200:
+            data = response.json()
+            return [
+                member['@odata.id'].split('/')[-1]
+                for member in data.get('Members', [])
+            ]
+    except (requests.exceptions.RequestException, ValueError, KeyError):
+        pass
+    return None
+
+
+def configure_server(
+    ip_address: str,
+    user: str,
+    password: str,
+    timeout: int,
+) -> Dict[str, Any]:
+    """Configure telemetry for a single server."""
+    session = requests.Session()
+    session.verify = False
+
+    try:
+        base_url = f"https://{ip_address}/redfish/v1/TelemetryService"
+
+        # Enable Telemetry Service
+        response = session.patch(
+            base_url,
+            json={"ServiceEnabled": True},
+            auth=(user, password),
+            timeout=timeout,
+        )
+
+        if response.status_code not in [200, 202, 204]:
+            return {
+                "ip": ip_address,
+                "status": "failed",
+                "message": f"Service HTTP {response.status_code}"
+            }
+
+        # Get available reports
+        available_reports = get_report_definitions(
+            ip_address, user, password, session, timeout
+        )
+        if not available_reports:
+            return {
+                "ip": ip_address,
+                "status": "failed",
+                "message": "Cannot get reports"
+            }
+
+        return {
+            "ip": ip_address,
+            "status": "success",
+            "enabled_reports": available_reports,
+        }
+
+    except requests.exceptions.RequestException as e:
+        return {
+            "ip": ip_address,
+            "status": "failed",
+            "message": str(e)
+        }
+
+    finally:
+        try:
+            session.close()
+        except OSError as close_error:
+            logging.warning("Warning: failed to close session for %s: %s", ip_address, close_error)
+
+def run_parallel(
+    idrac_ips: List[str],
+    username: str,
+    password: str,
+    parallel_jobs: int,
+    timeout: int,
+) -> Tuple[List[Dict], List[Dict]]:
+    """Run telemetry configuration in parallel."""
+    success_results = []
+    failed_results = []
+
+    try:
+        workers = max(1, min(os.cpu_count() + 1, parallel_jobs))
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            future_to_ip = {
+                executor.submit(
+                    configure_server, ip, username, password, timeout
+                ): ip for ip in idrac_ips
+            }
+
+            for future in as_completed(future_to_ip):
+                result = future.result()
+                if result.get("status") == "success":
+                    success_results.append(result)
+                else:
+                    failed_results.append(result)
+    except (OSError, ValueError, requests.exceptions.RequestException) as exc:
+        logging.warning("Error during parallel execution: %s", exc)
+
+    return success_results, failed_results
+
+def main():
+    """Main function for Ansible module."""
+    module_args = {
+        "idrac_ips": {"type": "list", "required": True, "elements": "str"},
+        "username": {"type": "str", "required": True},
+        "password": {"type": "str", "required": True, "no_log": True},
+        "parallel_jobs": {"type": "int", "default": 64},
+        "timeout": {"type": "int", "default": 30},
+    }
+
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+    idrac_ips = module.params["idrac_ips"]
+    username = module.params["username"]
+    password = module.params["password"]
+    parallel_jobs = module.params["parallel_jobs"]
+    timeout = module.params["timeout"]
+
+    if module.check_mode:
+        module.exit_json(changed=False, msg="Check mode - no changes made")
+
+    if not idrac_ips:
+        module.exit_json(msg="No iDRAC IPs provided")
+
+    start_time = time.time()
+    success_results, failed_results = run_parallel(
+        idrac_ips, username, password, parallel_jobs, timeout
+    )
+
+    duration = time.time() - start_time
+
+    module.exit_json(
+        changed=len(success_results) > 0,
+        success_count=len(success_results),
+        failed_count=len(failed_results),
+        duration_seconds=round(duration, 2),
+        success_results=success_results,
+        failed_results=failed_results,
+        msg=f"Telemetry enabled on {len(success_results)}/{len(idrac_ips)} servers"
+    )
+
+if __name__ == "__main__":
+    main()

common/vars/openchami_image_cmd.yml

@@ -20,6 +20,8 @@ rhel_aarch64_base_image_name: "rhel-aarch64_base"
 base_image_commands:
   - "dracut --add 'dmsquash-live livenet network-manager' --install '/usr/lib/systemd/systemd-sysroot-fstab-check' --kver $(basename /lib/modules/*) -N -f --logfile /tmp/dracut.log 2>/dev/null"   # noqa: yaml[line-length]
   - "echo DRACUT LOG:; cat /tmp/dracut.log"
+  - "rm -f /var/lib/rpm/__db*"
+  - "rpmdb --rebuilddb"
 
 #  x86_64 compute commands
 default_x86_64_compute_commands:

discovery/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2

@@ -53,7 +53,7 @@
                 done
             fi
 
-        - path: /root/install_cuda_toolkit.sh
+        - path: /usr/local/bin/install_cuda_toolkit.sh
           permissions: '0755'
           content: |
             #!/bin/bash
@@ -162,22 +162,29 @@
             {{ ip_name_map[key] }} {{ key }}
 {% endfor %}
 
+        - path: /usr/local/bin/check_slurm_controller_status.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}
 
       runcmd:
         - /usr/local/bin/set-ssh.sh
-        - /root/install_cuda_toolkit.sh
+        - /usr/local/bin/install_cuda_toolkit.sh
         - groupadd -r {{ slurm_group_name }}
         - useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}
 
-        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge
+        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /var/log/track
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  /var/log/slurm   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -a
         - yes | cp /etc/slurm/epilog.d/slurmd.service /usr/lib/systemd/system/
+        - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ user }}:{{ slurm_group_name }} /var/log/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/run/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/spool
@@ -307,3 +314,4 @@
 
         - /root/ldms_sampler.sh
 {% endif %}
+        - echo "Cloud-Init has completed successfully."

discovery/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2

@@ -54,7 +54,7 @@
             fi
 
 
-        - path: /root/install_cuda_toolkit.sh
+        - path: /usr/local/bin/install_cuda_toolkit.sh
           permissions: '0755'
           content: |
             #!/bin/bash
@@ -164,22 +164,30 @@
             {{ ip_name_map[key] }} {{ key }}
 {% endfor %}
 
+        - path: /usr/local/bin/check_slurm_controller_status.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}
 
       runcmd:
         - /usr/local/bin/set-ssh.sh
-        - /root/install_cuda_toolkit.sh
+        - /usr/local/bin/install_cuda_toolkit.sh
         - groupadd -r {{ slurm_group_name }}
         - useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}
 
-        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge
+        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /cert /var/log/track
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  /var/log/slurm   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -a
         - yes | cp /etc/slurm/epilog.d/slurmd.service /usr/lib/systemd/system/
+        - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ user }}:{{ slurm_group_name }} /var/log/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/run/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/spool
@@ -208,6 +216,8 @@
         - systemctl start slurmd
         - systemctl daemon-reexec
         - systemctl restart sshd
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
 
 {% if hostvars['localhost']['openldap_support'] %}
         - /usr/local/bin/update_ldap_conf.sh
@@ -309,3 +319,4 @@
 
         - /root/ldms_sampler.sh
 {% endif %}
+        - echo "Cloud-Init has completed successfully."

discovery/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2

@@ -85,20 +85,28 @@
             {{ ip_name_map[key] }} {{ key }}
 {% endfor %}
 
+        - path: /usr/local/bin/check_slurm_controller_status.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}
+
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - groupadd -r {{ slurm_group_name }}
         - useradd -r -g {{ slurm_group_name }} -d {{ home_dir }} -s /sbin/nologin {{ user }}
 
-        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge
+        - mkdir -p /var/log/slurm /var/run/slurm /var/spool /var/lib/slurm /etc/slurm/epilog.d /etc/munge /var/log/track
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  /var/log/slurm   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool      /var/spool       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -a
         - yes | cp /etc/slurm/epilog.d/slurmd.service /usr/lib/systemd/system/
+        - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ user }}:{{ slurm_group_name }} /var/log/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/run/slurm
         - chown -R {{ user }}:{{ slurm_group_name }} /var/spool
@@ -163,3 +171,4 @@
 
         - /root/ldms_sampler.sh
 {% endif %}
+        - echo "Cloud-Init has completed successfully."