diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bab7e79ef..0aa19b6261 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,7 +63,7 @@ repos: language: system entry: addlicense args: ['-c', '"Google LLC"', '-l', 'apache'] - exclude: docs/videos/healthcare-and-life-sciences/lysozyme-example/submit.sh + exclude: "docs/videos/healthcare-and-life-sciences/lysozyme-example/submit.sh|community/examples/slurm-gke/images/containers/.*" pass_filenames: true - id: tests-metadata name: tests-metadata @@ -111,6 +111,7 @@ repos: hooks: - id: yamllint args: [-c=.yamllint, --no-warnings] + exclude: community/examples/slurm-gke/images/containers/.* - repo: https://github.com/jackdewinter/pymarkdown rev: v0.9.17 hooks: @@ -129,7 +130,7 @@ repos: hooks: - id: script-must-have-extension - id: shellcheck - exclude: ".*unlinted" + exclude: ".*unlinted|community/examples/slurm-gke/images/containers/.*" - id: shfmt exclude: ".*tpl|.*unlinted" - repo: https://github.com/pre-commit/pre-commit-hooks @@ -140,5 +141,5 @@ repos: rev: v2.2.6 hooks: - id: codespell - exclude: requirements.txt$|/js/|go.sum$|cluster-toolkit-writers.json$ + exclude: requirements.txt$|/js/|go.sum$|cluster-toolkit-writers.json$|community/examples/slurm-gke/images/containers/.* exclude: tools/validate_configs/golden_copies/.* diff --git a/community/examples/slurm-gke/files/cgroup.conf.tpl b/community/examples/slurm-gke/files/cgroup.conf.tpl new file mode 100644 index 0000000000..c5e9927350 --- /dev/null +++ b/community/examples/slurm-gke/files/cgroup.conf.tpl @@ -0,0 +1,10 @@ +# cgroup.conf +# https://slurm.schedmd.com/cgroup.conf.html + +CgroupPlugin=autodetect +IgnoreSystemd=yes +# EnableControllers=yes +ConstrainCores=yes +ConstrainRamSpace=yes +ConstrainSwapSpace=no +ConstrainDevices=yes diff --git a/community/examples/slurm-gke/files/slurm-namespace.yaml.tftpl b/community/examples/slurm-gke/files/slurm-namespace.yaml.tftpl new file mode 100644 index 0000000000..ec18036340 --- /dev/null +++ b/community/examples/slurm-gke/files/slurm-namespace.yaml.tftpl @@ -0,0 +1,18 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/community/examples/slurm-gke/images/containers/LICENSES/Apache-2.0.txt b/community/examples/slurm-gke/images/containers/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/community/examples/slurm-gke/images/containers/LICENSES/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/community/examples/slurm-gke/images/containers/Makefile b/community/examples/slurm-gke/images/containers/Makefile new file mode 100644 index 0000000000..9923db3255 --- /dev/null +++ b/community/examples/slurm-gke/images/containers/Makefile @@ -0,0 +1,89 @@ +# --- Configuration --- +# OS variant (e.g., ubuntu24.04, rockylinux9) +OS ?= ubuntu24.04 +# Slurm version, also used for the directory path +SLURM_VERSION ?= 25.05 + +# Container registry and repository +REGISTRY ?= +REPO ?= + +# --- Validation --- +# Ensure required variables are set before proceeding. +ifeq ($(strip $(REGISTRY)),) + $(error REGISTRY is not set. Please provide your container registry. \ + Example: make REGISTRY=gcr.io/my-project ...) +endif + +ifeq ($(strip $(REPO)),) + $(error REPO is not set. Please provide your repository name. \ + Example: make REPO=my-image-repo ...) +endif + +# Build arguments for Pyxis +ENROOT_VERSION ?= 3.5.0 +PYXIS_VERSION ?= 0.20.0 + + +# --- Path and Image Naming --- +# Path to the build context directories +BUILD_PATH := schedmd/slurm/$(SLURM_VERSION) + +# Image tag +TAG ?= $(SLURM_VERSION)-$(OS) + +# Defines the full names for target images +SLURMD_IMAGE := $(REGISTRY)/$(REPO)/slurmd:$(TAG) +SLURMD_PYXIS_IMAGE := $(REGISTRY)/$(REPO)/slurmd-pyxis:$(TAG) + + +# --- Targets --- +.PHONY: all build push clean +.PHONY: build-slurmd build-slurmd-pyxis +.PHONY: push-slurmd push-slurmd-pyxis + +all: build + +build: build-slurmd build-slurmd-pyxis + @echo "All images built successfully." + +push: push-slurmd push-slurmd-pyxis + @echo "All images pushed successfully." + + +# --- Individual Build Targets --- +build-slurmd: + @echo "Building $(SLURMD_IMAGE) with buildx..." + docker buildx build \ + --file $(BUILD_PATH)/$(OS)/Dockerfile \ + --target slurmd \ + --progress=plain \ + --tag $(SLURMD_IMAGE) \ + $(BUILD_PATH)/$(OS)/ + +build-slurmd-pyxis: build-slurmd + @echo "Building $(SLURMD_PYXIS_IMAGE) with buildx..." + docker buildx build \ + --file $(BUILD_PATH)/$(OS)/Dockerfile.pyxis \ + --target slurmd-pyxis \ + --build-arg ENROOT_VERSION=$(ENROOT_VERSION) \ + --build-arg PYXIS_VERSION=$(PYXIS_VERSION) \ + --build-arg BASE_CONTAINER_PATH=$(SLURMD_IMAGE) \ + --tag $(SLURMD_PYXIS_IMAGE) \ + $(BUILD_PATH)/$(OS)/ + +# --- Individual Push Targets --- +push-slurmd: + @echo "Pushing $(SLURMD_IMAGE)..." + docker push $(SLURMD_IMAGE) + +push-slurmd-pyxis: + @echo "Pushing $(SLURMD_PYXIS_IMAGE)..." + docker push $(SLURMD_PYXIS_IMAGE) + + +# --- Housekeeping --- +clean: + @echo "Cleaning up local images..." + -docker rmi $(SLURMD_IMAGE) || true + -docker rmi $(SLURMD_PYXIS_IMAGE) || true diff --git a/community/examples/slurm-gke/images/containers/README.md b/community/examples/slurm-gke/images/containers/README.md new file mode 100644 index 0000000000..831c1d22f1 --- /dev/null +++ b/community/examples/slurm-gke/images/containers/README.md @@ -0,0 +1,110 @@ +# Slinky container image for GKE + +This is a modified version of the original [Slinky Containers](https://github.com/SlinkyProject/slinky-containers) by SchedMD. This fork has been adapted for running on Google Kubernetes Engine(GKE). + +## Prerequisites + +Before you begin, ensure you have the following installed and configured: + +* **Docker:** with the `buildx` plugin enabled. +* **Google Cloud SDK (`gcloud`):** for interacting with Google Artifact Registry. +* **Authentication:** You must be authenticated with Google Cloud and have configured Docker to use your credentials. + + ```bash + # Authenticate with gcloud + gcloud auth login + + # Configure Docker credentials for Artifact Registry (replace with your region) + gcloud auth configure-docker -docker.pkg.dev + ``` + +*** + +## 1. Setup Google Artifact Registry + +You need a Docker repository in Google Artifact Registry to store your images. + +1. **Choose a repository name** (e.g., `slurm-images`) and a **location** (e.g., `us-west1`). +2. Run the following `gcloud` command to create the repository: + + ```bash + gcloud artifacts repositories create \ + --repository-format=docker \ + --location= \ + --description="Docker repository for Slurm images" + ``` + + *Replace `` and `` with your chosen values.* + +*** + +## 2. Configure the Makefile + +You **must** update the `Makefile` to point to your Artifact Registry repository. + +1. Open the `Makefile` in a text editor. +2. Locate the `REGISTRY` and `REPO` variables. +3. Update them with your GCP project ID, location, and the repository name you just created. + + **Example:** + If your project ID is `my-hpc-project`, your location is `us-west1`, and your repository name is `slurm-images`, the configuration should look like this: + + ```makefile + # Container registry and repository + REGISTRY ?= us-west1-docker.pkg.dev/my-hpc-project + REPO ?= slurm-images + ``` + +*** + +## 3. Build and Push the Images + +The `Makefile` provides several targets to build and push the images individually or all at once. + +### Build Images + +* **Build both `slurmd` and `slurmd-pyxis` images:** + + ```bash + make build + ``` + +* **Build only the base `slurmd` image:** + + ```bash + make build-slurmd + ``` + +* **Build the `slurmd-pyxis` image** (this will also build the base `slurmd` image first as it's a dependency): + + ```bash + make build-slurmd-pyxis + ``` + +### Push Images + +After building, you can push the images to your configured Artifact Registry. + +* **Push both images:** + + ```bash + make push + ``` + +* **Push only the `slurmd` image:** + + ```bash + make push-slurmd + ``` + +* **Push only the `slurmd-pyxis` image:** + + ```bash + make push-slurmd-pyxis + ``` + +## License + +This project is licensed under the Apache License, Version 2.0. The original copyright belongs to SchedMD LLC. My modifications are also licensed under the same terms. + +A full copy of the license is available in the [LICENSE](./LICENSE) file. diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/Dockerfile b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/Dockerfile new file mode 100644 index 0000000000..295ea000da --- /dev/null +++ b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/Dockerfile @@ -0,0 +1,167 @@ +# syntax=docker/dockerfile:1 +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-FileCopyrightText: Copyright (C) 2025 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +################################################################################ + +ARG SLURM_VERSION=25.05-latest +ARG SLURM_DIR="slurm-${SLURM_VERSION}" +ARG PARENT_IMAGE=rockylinux:9 + +################################################################################ +FROM alpine:latest AS slurm-src + +ARG SLURM_VERSION +ARG SLURM_DIR + +WORKDIR /workspace/ + +ADD https://download.schedmd.com/slurm/${SLURM_DIR}.tar.bz2 ${SLURM_DIR}.tar.bz2 + +RUN </dev/null +## Cleanup +dnf clean all +EOR + +################################################################################ +FROM ${PARENT_IMAGE} AS base + +SHELL ["bash", "-c"] + +ARG SLURM_VERSION +ENV SLURM_VERSION=${SLURM_VERSION} + +USER root +WORKDIR /tmp/ + +ARG SLURM_USER=slurm +ARG SLURM_USER_UID=981 +ARG SLURM_USER_GID=981 + +RUN </]slurmd: .` +################################################################################ +FROM base AS slurmd + +SHELL ["bash", "-c"] + +USER root +WORKDIR /tmp/ + +# Ref: https://slurm.schedmd.com/quickstart_admin.html#pkg_install +RUN </]slurmd-pyxis: .` +################################################################################ +FROM ${BASE_CONTAINER_PATH} AS slurmd-pyxis + +ARG DEBIAN_FRONTEND=noninteractive + +ARG ENROOT_VERSION +ENV ENROOT_VERSION="${ENROOT_VERSION}" + +ARG PYXIS_VERSION +ENV PYXIS_VERSION="${PYXIS_VERSION}" + +COPY --from=download /tmp/*.rpm ./ +COPY --from=download /tmp/**/*.rpm ./ + +# Ref: https://github.com/NVIDIA/pyxis?tab=readme-ov-file#with-a-rpm-package +# Ref: https://github.com/NVIDIA/enroot/blob/master/doc/installation.md +RUN < 0)); then + echo "$coreSpecCount" + else + echo "0" + fi +} + +# calculateMemSpecLimit returns a value for MemSpecLimit for the pod. +# +# MemSpecLimit represents the amount of memory that the slurmd/slurmstepd +# cannot use. Effectively it is the difference of the host and the pod's +# resource limits. We have to convert memory to MB. +# +# Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_MemSpecLimit +function calculateMemSpecLimit() { + local memSpecLimit=0 + local totalMemory=0 + + totalMemory="$(gawk '/^MemTotal:/{ print $2 }' /proc/meminfo)" + memSpecLimit="$(((totalMemory / 1024) - POD_MEMORY))" + + if ((memSpecLimit > 0)); then + echo "$memSpecLimit" + else + echo "0" + fi +} + +# addConfItem shims the item into SLURMD_OPTIONS. +# +# This function will add `--conf` if it is not present in SLURMD_OPTIONS, +# otherwise will add the item into the argument of `--conf`. +function addConfItem() { + local item="$1" + local slurmdOptions=() + local foundConf=0 + readarray -t slurmdOptions < <(echo -n "$SLURMD_OPTIONS" | gawk -v FPAT="([^ ]+)|[^ ]*((\"[^\"]+\")|('[^']+'))" '{ for (i=1; i<=NF; i++) print $i }') + for i in "${!slurmdOptions[@]}"; do + case "${slurmdOptions[$i]}" in + --conf=*) + foundConf=1 + local val="${slurmdOptions[$i]#--conf=}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$i]="--conf='${val} ${item}'" + ;; + --conf) + foundConf=1 + local j="$((i + 1))" + local val="${slurmdOptions[$j]}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$j]="'${val} ${item}'" + ;; + *) ;; + esac + done + if ((foundConf == 0)); then + slurmdOptions+=("--conf") + slurmdOptions+=("'${item}'") + fi + export SLURMD_OPTIONS="${slurmdOptions[*]}" +} + +function main() { + mkdir -p /run/slurm/ + mkdir -p /var/spool/slurmd/ + + local coreSpecCount=0 + if ((POD_CPUS > 0)); then + coreSpecCount="$(calculateCoreSpecCount)" + fi + if ((coreSpecCount > 0)); then + addConfItem "CoreSpecCount=${coreSpecCount}" + fi + + local memSpecLimit=0 + if ((POD_MEMORY > 0)); then + memSpecLimit="$(calculateMemSpecLimit)" + fi + if ((memSpecLimit > 0)); then + addConfItem "MemSpecLimit=${memSpecLimit}" + fi + + exec supervisord -c /etc/supervisor/supervisord.conf +} +main diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/etc/supervisor/supervisord.conf b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/etc/supervisor/supervisord.conf new file mode 100644 index 0000000000..7ae56e1a45 --- /dev/null +++ b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/etc/supervisor/supervisord.conf @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-License-Identifier: Apache-2.0 + +# Ref: https://supervisord.org/configuration.html +[supervisord] +nodaemon=true +user=root +logfile=/dev/null +logfile_maxbytes=0 + +[program:slurmd] +environment=NOTIFY_SOCKET="/dev/null" +command=bash -xc "exec slurmd --systemd %(ENV_SLURMD_OPTIONS)s" +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 +stopasgroup=true + +[eventlistener:processes] +command=bash -c "printf 'READY\n' && while read line; do kill -SIGQUIT $PPID; done < /dev/stdin" +events=PROCESS_STATE_STOPPED,PROCESS_STATE_FATAL diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/usr/local/bin/entrypoint.sh b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/usr/local/bin/entrypoint.sh new file mode 100644 index 0000000000..ba4dd13fd5 --- /dev/null +++ b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/files/slurmd/usr/local/bin/entrypoint.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Additional arguments to pass to slurmd. +export SLURMD_OPTIONS="${SLURMD_OPTIONS:-} $*" + +# The asserted CPU resource limit of the pod. +export POD_CPUS="${POD_CPUS:-0}" + +# The asserted memory resource limit (in MB) of the pod. +export POD_MEMORY="${POD_MEMORY:-0}" + +# calculateCoreSpecCount returns a value for CoreSpecCount for the pod. +# +# CoreSpecCount represents the number of cores that the slurmd/slurmstepd +# should not use. Effectively it is the difference of the host and the pod's +# resource limits. We have to convert CPUs to cores. +# +# Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_CoreSpecCount +# Ref: https://slurm.schedmd.com/core_spec.html +function calculateCoreSpecCount() { + local coreSpecCount=0 + local coreCount=0 + local threadCount=0 + + coreCount="$(($(lscpu | gawk '/^Socket\(s\):/{ print $2 }') * $(lscpu | gawk '/^Core\(s\) per socket:/{ print $4 }')))" + threadCount="$(lscpu | gawk '/^Thread\(s\) per core:/{ print $4 }')" + coreSpecCount="$((coreCount - (POD_CPUS / threadCount)))" + + if ((coreSpecCount > 0)); then + echo "$coreSpecCount" + else + echo "0" + fi +} + +# calculateMemSpecLimit returns a value for MemSpecLimit for the pod. +# +# MemSpecLimit represents the amount of memory that the slurmd/slurmstepd +# cannot use. Effectively it is the difference of the host and the pod's +# resource limits. We have to convert memory to MB. +# +# Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_MemSpecLimit +function calculateMemSpecLimit() { + local memSpecLimit=0 + local totalMemory=0 + + totalMemory="$(gawk '/^MemTotal:/{ print $2 }' /proc/meminfo)" + memSpecLimit="$(((totalMemory / 1024) - POD_MEMORY))" + + if ((memSpecLimit > 0)); then + echo "$memSpecLimit" + else + echo "0" + fi +} + +# addConfItem shims the item into SLURMD_OPTIONS. +# +# This function will add `--conf` if it is not present in SLURMD_OPTIONS, +# otherwise will add the item into the argument of `--conf`. +function addConfItem() { + local item="$1" + local slurmdOptions=() + local foundConf=0 + readarray -t slurmdOptions < <(echo -n "$SLURMD_OPTIONS" | gawk -v FPAT="([^ ]+)|[^ ]*((\"[^\"]+\")|('[^']+'))" '{ for (i=1; i<=NF; i++) print $i }') + for i in "${!slurmdOptions[@]}"; do + case "${slurmdOptions[$i]}" in + --conf=*) + foundConf=1 + local val="${slurmdOptions[$i]#--conf=}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$i]="--conf='${val} ${item}'" + ;; + --conf) + foundConf=1 + local j="$((i + 1))" + local val="${slurmdOptions[$j]}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$j]="'${val} ${item}'" + ;; + *) ;; + esac + done + if ((foundConf == 0)); then + slurmdOptions+=("--conf") + slurmdOptions+=("'${item}'") + fi + export SLURMD_OPTIONS="${slurmdOptions[*]}" +} + +function main() { + mkdir -p /run/slurm/ + mkdir -p /var/spool/slurmd/ + + local coreSpecCount=0 + if ((POD_CPUS > 0)); then + coreSpecCount="$(calculateCoreSpecCount)" + fi + if ((coreSpecCount > 0)); then + addConfItem "CoreSpecCount=${coreSpecCount}" + fi + + local memSpecLimit=0 + if ((POD_MEMORY > 0)); then + memSpecLimit="$(calculateMemSpecLimit)" + fi + if ((memSpecLimit > 0)); then + addConfItem "MemSpecLimit=${memSpecLimit}" + fi + cat >/etc/enroot/enroot.conf <<'EOF' + ENROOT_RUNTIME_PATH /run/enroot/${UID}/run + ENROOT_CONFIG_PATH /run/enroot/${UID}/config + ENROOT_CACHE_PATH /run/enroot/${UID}/cache + ENROOT_DATA_PATH /run/enroot/${UID}/data + ENROOT_TEMP_PATH /run/${UID}/tmp +EOF + exec supervisord -c /etc/supervisor/supervisord.conf +} +main diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/patches/.gitkeep b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/rockylinux9/patches/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/Dockerfile b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/Dockerfile new file mode 100644 index 0000000000..bf2b5fbbee --- /dev/null +++ b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/Dockerfile @@ -0,0 +1,164 @@ +# syntax=docker/dockerfile:1 +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-FileCopyrightText: Copyright (C) 2025 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +################################################################################ + +ARG SLURM_VERSION=25.05-latest +ARG SLURM_DIR="slurm-${SLURM_VERSION}" +ARG PARENT_IMAGE=ubuntu:24.04 + +################################################################################ +FROM alpine:latest AS slurm-src + +ARG SLURM_VERSION +ARG SLURM_DIR + +WORKDIR /workspace/ + +ADD https://download.schedmd.com/slurm/${SLURM_DIR}.tar.bz2 ${SLURM_DIR}.tar.bz2 + +RUN </dev/null ) +## Cleanup +apt-get clean && rm -rf /var/lib/apt/lists/* +EOR + +################################################################################ +FROM ${PARENT_IMAGE} AS base + +SHELL ["bash", "-c"] + +ARG DEBIAN_FRONTEND=noninteractive + +ARG SLURM_VERSION +ENV SLURM_VERSION=${SLURM_VERSION} + +USER root +WORKDIR /tmp/ + +ARG SLURM_USER=slurm +ARG SLURM_USER_UID=981 +ARG SLURM_USER_GID=981 + +RUN </]slurmd: .` +################################################################################ +FROM base AS slurmd + +SHELL ["bash", "-c"] + +ARG DEBIAN_FRONTEND=noninteractive + +USER root +WORKDIR /tmp/ + +# Ref: https://slurm.schedmd.com/quickstart_admin.html#pkg_install +RUN </]slurmd-pyxis: .` +################################################################################ +FROM ${BASE_CONTAINER_PATH} as slurmd-pyxis + +ARG DEBIAN_FRONTEND=noninteractive + +ARG ENROOT_VERSION +ENV ENROOT_VERSION="${ENROOT_VERSION}" + +ARG PYXIS_VERSION +ENV PYXIS_VERSION="${PYXIS_VERSION}" + +COPY --from=download /tmp/*.deb ./ + +# Ref: https://github.com/NVIDIA/pyxis?tab=readme-ov-file#with-a-deb-package +# Ref: https://github.com/NVIDIA/enroot/blob/master/doc/installation.md +RUN < 0)); then + echo "$coreSpecCount" + else + echo "0" + fi +} + +# calculateMemSpecLimit returns a value for MemSpecLimit for the pod. +# +# MemSpecLimit represents the amount of memory that the slurmd/slurmstepd +# cannot use. Effectively it is the difference of the host and the pod's +# resource limits. We have to convert memory to MB. +# +# Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_MemSpecLimit +function calculateMemSpecLimit() { + local memSpecLimit=0 + local totalMemory=0 + + totalMemory="$(gawk '/^MemTotal:/{ print $2 }' /proc/meminfo)" + memSpecLimit="$(((totalMemory / 1024) - POD_MEMORY))" + + if ((memSpecLimit > 0)); then + echo "$memSpecLimit" + else + echo "0" + fi +} + +# addConfItem shims the item into SLURMD_OPTIONS. +# +# This function will add `--conf` if it is not present in SLURMD_OPTIONS, +# otherwise will add the item into the argument of `--conf`. +function addConfItem() { + local item="$1" + local slurmdOptions=() + local foundConf=0 + readarray -t slurmdOptions < <(echo -n "$SLURMD_OPTIONS" | gawk -v FPAT="([^ ]+)|[^ ]*((\"[^\"]+\")|('[^']+'))" '{ for (i=1; i<=NF; i++) print $i }') + for i in "${!slurmdOptions[@]}"; do + case "${slurmdOptions[$i]}" in + --conf=*) + foundConf=1 + local val="${slurmdOptions[$i]#--conf=}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$i]="--conf='${val} ${item}'" + ;; + --conf) + foundConf=1 + local j="$((i + 1))" + local val="${slurmdOptions[$j]}" + val="$(echo -n "$val" | sed -e 's/[\\]*"//g' -e "s/[\\]*'//g")" + slurmdOptions[$j]="'${val} ${item}'" + ;; + *) ;; + esac + done + if ((foundConf == 0)); then + slurmdOptions+=("--conf") + slurmdOptions+=("'${item}'") + fi + export SLURMD_OPTIONS="${slurmdOptions[*]}" +} + +function main() { + mkdir -p /run/slurm/ + mkdir -p /var/spool/slurmd/ + + local coreSpecCount=0 + if ((POD_CPUS > 0)); then + coreSpecCount="$(calculateCoreSpecCount)" + fi + if ((coreSpecCount > 0)); then + addConfItem "CoreSpecCount=${coreSpecCount}" + fi + + local memSpecLimit=0 + if ((POD_MEMORY > 0)); then + memSpecLimit="$(calculateMemSpecLimit)" + fi + if ((memSpecLimit > 0)); then + addConfItem "MemSpecLimit=${memSpecLimit}" + fi + + cat >/etc/enroot/enroot.conf <<'EOF' + ENROOT_RUNTIME_PATH /run/enroot/${UID}/run + ENROOT_CONFIG_PATH /run/enroot/${UID}/config + ENROOT_CACHE_PATH /run/enroot/${UID}/cache + ENROOT_DATA_PATH /run/enroot/${UID}/data + ENROOT_TEMP_PATH /run/${UID}/tmp +EOF + RUN chmod +r+x / && chmod +r+x /usr/ && chmod +r+x /usr/bin/ && chmod +r+x /etc/ && chmod +r+x /usr/local/ + exec supervisord -c /etc/supervisor/supervisord.conf +} +main diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/files/slurmd/etc/supervisor/supervisord.conf b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/files/slurmd/etc/supervisor/supervisord.conf new file mode 100644 index 0000000000..7ae56e1a45 --- /dev/null +++ b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/files/slurmd/etc/supervisor/supervisord.conf @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-License-Identifier: Apache-2.0 + +# Ref: https://supervisord.org/configuration.html +[supervisord] +nodaemon=true +user=root +logfile=/dev/null +logfile_maxbytes=0 + +[program:slurmd] +environment=NOTIFY_SOCKET="/dev/null" +command=bash -xc "exec slurmd --systemd %(ENV_SLURMD_OPTIONS)s" +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 +stopasgroup=true + +[eventlistener:processes] +command=bash -c "printf 'READY\n' && while read line; do kill -SIGQUIT $PPID; done < /dev/stdin" +events=PROCESS_STATE_STOPPED,PROCESS_STATE_FATAL diff --git a/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/patches/.gitkeep b/community/examples/slurm-gke/images/containers/schedmd/slurm/25.05/ubuntu24.04/patches/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/community/examples/slurm-gke/slurm-gke.yaml b/community/examples/slurm-gke/slurm-gke.yaml new file mode 100644 index 0000000000..c9ca639cf7 --- /dev/null +++ b/community/examples/slurm-gke/slurm-gke.yaml @@ -0,0 +1,174 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +blueprint_name: slurm-gke + +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: slurm-gke + region: us-west4 + zone: us-west4-c + authorized_cidr: 0.0.0.0/0 # /32 + gcp_public_cidrs_access_enabled: false + gke_nodeset_replicas: 2 + slinky_image: ghcr.io/slinkyproject/slurmd-pyxis:24.11-ubuntu24.04 + slurm_namespace: slurm + +deployment_groups: +- group: primary + modules: + +###### Common resources ###### + + - id: network + source: modules/network/vpc + settings: + subnetwork_name: $(vars.deployment_name)-subnet + secondary_ranges_list: + - subnetwork_name: $(vars.deployment_name)-subnet + ranges: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + + - id: homefs + source: modules/file-system/filestore + use: [network, private_service_access] + settings: + local_mount: /home + +###### GKE Setup ###### + + - id: gke_service_account + source: community/modules/project/service-account + settings: + name: slinky-gke-sa + project_roles: + - logging.logWriter + - monitoring.metricWriter + - monitoring.viewer + - stackdriver.resourceMetadata.writer + - storage.objectAdmin + - artifactregistry.reader + + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network, gke_service_account] + settings: + enable_private_endpoint: false + gcp_public_cidrs_access_enabled: $(vars.gcp_public_cidrs_access_enabled) + enable_gcsfuse_csi: true + enable_filestore_csi: true + master_authorized_networks: + - display_name: deployment-machine + cidr_block: $(vars.authorized_cidr) + system_node_pool_enabled: false + configure_workload_identity_sa: true + enable_dcgm_monitoring: true + outputs: [instructions] + + - id: gke_base_pool + source: modules/compute/gke-node-pool + use: [gke_cluster, gke_service_account] + settings: + initial_node_count: 1 + disk_type: pd-balanced + machine_type: e2-standard-4 + zones: [$(vars.zone)] + + - id: gke_compute_pool + source: modules/compute/gke-node-pool + use: [gke_cluster, gke_service_account] + settings: + name: gke-compute-pool + initial_node_count: $(vars.gke_nodeset_replicas) + disk_type: pd-balanced + machine_type: c2-standard-16 + zones: [$(vars.zone)] + + - id: gke_ns_manifest + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + apply_manifests: + - source: $(ghpc_stage("./files/slurm-namespace.yaml.tftpl")) + template_vars: + namespace: $(vars.slurm_namespace) + + - id: slinky + source: community/modules/scheduler/slinky + use: + - gke_cluster + - gke_base_pool # Optionally specify nodepool(s) to avoid operator components running on HPC hardware + settings: + slurm_operator_namespace: $(vars.slurm_namespace) + install_slurm_operator_chart: true + install_slurm_chart: false + + - id: gke_compute_nodeset + source: community/modules/compute/gke-nodeset + use: [gke_compute_pool, slinky, homefs, slurm_controller, network] + settings: + slurm_cluster_name: $(vars.deployment_name) + image: $(vars.slinky_image) + + - id: gke_compute_partition + source: community/modules/compute/gke-partition + use: [slurm_controller, gke_compute_nodeset] + +###### GCE Setup ###### + + - id: debug_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + node_count_dynamic_max: 4 + machine_type: n2-standard-2 + allow_automatic_updates: false + + - id: debug_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - debug_nodeset + settings: + partition_name: debug + exclusive: false # allows nodes to stay up after jobs are done + is_default: true + suspend_time: -1 # prevents nodes from suspending while it's idle + + - id: slurm_login + source: community/modules/scheduler/schedmd-slurm-gcp-v6-login + use: [network] + settings: + machine_type: n2-standard-4 + enable_login_public_ips: true + + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - slurm_login + - debug_partition + - homefs + settings: + slurm_cluster_name: $(vars.deployment_name) + enable_slurm_auth: true + cgroup_conf_tpl: $(ghpc_stage("./files/cgroup.conf.tpl")) + enable_controller_public_ips: true diff --git a/community/modules/compute/gke-nodeset/README.md b/community/modules/compute/gke-nodeset/README.md new file mode 100644 index 0000000000..0ee685d93e --- /dev/null +++ b/community/modules/compute/gke-nodeset/README.md @@ -0,0 +1,55 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | +| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | +| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | +| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | +| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | +| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | +| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | + diff --git a/community/modules/compute/gke-nodeset/main.tf b/community/modules/compute/gke-nodeset/main.tf new file mode 100644 index 0000000000..8b2f1deeac --- /dev/null +++ b/community/modules/compute/gke-nodeset/main.tf @@ -0,0 +1,64 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### GKE NodeSet +locals { + manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" +} + +module "kubectl_apply" { + source = "../../../../modules/management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = [{ + source = local.manifest_path, + template_vars = { + slurm_namespace = var.slurm_namespace, + nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + controller_name = "${var.slurm_cluster_name}-controller", + node_pool_name = var.node_pool_names[0], + node_count = var.node_count_static, + image = var.image, + home_pvc = module.home_pv.pvc_name + slurm_key_pvc = module.slurm_key_pv.pvc_name + } + }] +} + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm NodeSet +locals { + nodeset = { + gke_nodepool = var.node_pool_names[0] + nodeset_name = var.nodeset_name + node_count_static = var.node_count_static + subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" + instance_template = var.instance_templates[0] + } +} + +resource "google_storage_bucket_object" "gke_nodeset_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" + content = yamlencode(local.nodeset) +} diff --git a/community/modules/compute/gke-nodeset/metadata.yaml b/community/modules/compute/gke-nodeset/metadata.yaml new file mode 100644 index 0000000000..ea2cfc221e --- /dev/null +++ b/community/modules/compute/gke-nodeset/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com + - storage.googleapis.com diff --git a/community/modules/compute/gke-nodeset/output.tf b/community/modules/compute/gke-nodeset/output.tf new file mode 100644 index 0000000000..15970ff0b7 --- /dev/null +++ b/community/modules/compute/gke-nodeset/output.tf @@ -0,0 +1,18 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_name" { + description = "Name of the new Slinky nodset" + value = local.nodeset.nodeset_name +} diff --git a/community/modules/compute/gke-nodeset/persistent_volumes.tf b/community/modules/compute/gke-nodeset/persistent_volumes.tf new file mode 100644 index 0000000000..8a190c4019 --- /dev/null +++ b/community/modules/compute/gke-nodeset/persistent_volumes.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + slurm_key_storage = { + server_ip = var.slurm_controller_instance.network_interface[0].network_ip + remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py + client_install_runner = {} + mount_runner = {} + fs_type = "" + local_mount = "" + mount_options = "" + } +} + +module "slurm_key_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1 + cluster_id = var.cluster_id + filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore + namespace = var.slurm_namespace + network_storage = local.slurm_key_storage + pv_name = "slurm-key-pv" + pvc_name = "slurm-key-pvc" +} + +# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. +module "home_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1024 + cluster_id = var.cluster_id + filestore_id = var.filestore_id[0] + network_storage = var.network_storage[0] + namespace = var.slurm_namespace + pv_name = "home-pv" + pvc_name = "home-pvc" +} diff --git a/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl new file mode 100644 index 0000000000..a5a4a5e7ac --- /dev/null +++ b/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl @@ -0,0 +1,203 @@ +apiVersion: slinky.slurm.net/v1alpha1 +kind: NodeSet +metadata: + annotations: + meta.helm.sh/release-name: slurm + meta.helm.sh/release-namespace: ${slurm_namespace} + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + name: ${nodeset_name} + namespace: ${slurm_namespace} +spec: + clusterName: slurm + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + replicas: ${node_count} + revisionHistoryLimit: 0 + selector: + matchLabels: + app.kubernetes.io/instance: slurm + app.kubernetes.io/name: slurmd + nodeset.slinky.slurm.net/name: ${nodeset_name} + serviceName: slurm-compute + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: slurmd + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + spec: + automountServiceAccountToken: false + containers: + - args: + - -g + - -- + - bash + - -c + - | + mkdir -p /usr/local/lib/slurm + ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so + /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME + command: + - tini + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_CPUS + value: "0" + - name: POD_MEMORY + value: "0" + image: ${image} + imagePullPolicy: IfNotPresent + name: slurmd + ports: + - containerPort: 6818 + name: slurmd + protocol: TCP + readinessProbe: + exec: + command: + - scontrol + - show + - slurmd + resources: {} + securityContext: + capabilities: + add: + - BPF + - NET_ADMIN + - SYS_ADMIN + - SYS_NICE + privileged: true + volumeMounts: + - mountPath: /etc/slurm + name: etc-slurm + - mountPath: /run + name: run + - mountPath: /var/spool/slurmd + name: slurm-spool + - mountPath: /var/log/slurm + name: slurm-log + - mountPath: /home + name: home-pvc + dnsConfig: + searches: + - ${controller_name} + hostNetwork: true + initContainers: + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal + dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir + -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod + -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# + Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# + Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type + f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs + -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind + \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind + \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" + -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 + | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal + dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; + then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm + -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" + >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# + Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction + main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" + env: + - name: SLURM_USER + value: slurm + image: ${image} + imagePullPolicy: IfNotPresent + name: init + resources: {} + volumeMounts: + - mountPath: /mnt/slurm + name: slurm-config + - mountPath: /mnt/etc/slurm + name: etc-slurm + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname + \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo + -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone + <\"$SOCKET\"\n" + env: + - name: SOCKET + value: /var/log/slurm/slurmd.log + image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 + imagePullPolicy: IfNotPresent + name: logfile + resources: {} + restartPolicy: Always + volumeMounts: + - mountPath: /var/log/slurm + name: slurm-log + nodeSelector: + cloud.google.com/gke-nodepool: ${node_pool_name} + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Equal + value: present + volumes: + - emptyDir: + medium: Memory + name: etc-slurm + - emptyDir: {} + name: run + - name: slurm-config + persistentVolumeClaim: + claimName: ${slurm_key_pvc} + - emptyDir: + medium: Memory + name: slurm-spool + - emptyDir: + medium: Memory + name: slurm-log + - name: home-pvc + persistentVolumeClaim: + claimName: ${home_pvc} + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/community/modules/compute/gke-nodeset/variables.tf b/community/modules/compute/gke-nodeset/variables.tf new file mode 100644 index 0000000000..c091a0da86 --- /dev/null +++ b/community/modules/compute/gke-nodeset/variables.tf @@ -0,0 +1,118 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name, used in slurm controller" + + validation { + condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "slurm_controller_instance" { + type = any + description = "Slurm cluster controller instance" +} + +variable "image" { + description = "The image for slurm daemon" + type = string + nullable = false +} + +variable "node_pool_names" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = list(string) + nullable = false +} + +variable "node_count_static" { + description = "The number of static nodes in node-pool" + type = number +} + +variable "subnetwork" { + description = "Primary subnetwork object" + type = any +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} + +variable "instance_templates" { + description = "The URLs of Instance Templates" + type = list(string) + nullable = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + + validation { + condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" + error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." + } +} + +variable "filestore_id" { + description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = list(string) + + validation { + condition = length(var.filestore_id) == 1 + error_message = "The 'filestore_id' variable must contain exactly one element." + } +} diff --git a/community/modules/compute/gke-nodeset/versions.tf b/community/modules/compute/gke-nodeset/versions.tf new file mode 100644 index 0000000000..3d7237cb92 --- /dev/null +++ b/community/modules/compute/gke-nodeset/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" + } +} diff --git a/community/modules/compute/gke-partition/README.md b/community/modules/compute/gke-partition/README.md new file mode 100644 index 0000000000..2a7c363a87 --- /dev/null +++ b/community/modules/compute/gke-partition/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/community/modules/compute/gke-partition/main.tf b/community/modules/compute/gke-partition/main.tf new file mode 100644 index 0000000000..2949fd6594 --- /dev/null +++ b/community/modules/compute/gke-partition/main.tf @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm Partition +locals { + partition_conf = { + "PowerDownOnIdle" = "NO" + "SuspendTime" = "INFINITE" + "SuspendTimeout" = var.has_tpu ? 240 : 120 + "ResumeTimeout" = var.has_tpu ? 600 : 300 + } + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [var.nodeset_name] + partition_nodeset_tpu = [] + partition_nodeset_dyn = [] + # Options + enable_job_exclusive = true + power_down_on_idle = false + } +} + +resource "google_storage_bucket_object" "parition_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" + content = yamlencode(local.partition) +} diff --git a/community/modules/compute/gke-partition/metadata.yaml b/community/modules/compute/gke-partition/metadata.yaml new file mode 100644 index 0000000000..557e1fc2ae --- /dev/null +++ b/community/modules/compute/gke-partition/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/community/modules/compute/gke-partition/variables.tf b/community/modules/compute/gke-partition/variables.tf new file mode 100644 index 0000000000..3aeed2e59a --- /dev/null +++ b/community/modules/compute/gke-partition/variables.tf @@ -0,0 +1,43 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "has_tpu" { + description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." + type = bool + default = false +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "partition_name" { + description = "The partition name" + type = string + default = "gke" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} diff --git a/community/modules/compute/gke-partition/versions.tf b/community/modules/compute/gke-partition/versions.tf new file mode 100644 index 0000000000..aede55263c --- /dev/null +++ b/community/modules/compute/gke-partition/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" + } +} diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md index d0fa762e5a..da8b0ca858 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -395,6 +395,7 @@ limitations under the License. | Name | Description | |------|-------------| | [instructions](#output\_instructions) | Post deployment instructions. | +| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | | [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | | [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | | [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py index 35c478c1a5..217fd0bca2 100755 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py @@ -1695,15 +1695,10 @@ def node_is_dyn(self, node_name=None) -> bool: return self.cfg.nodeset_dyn.get(nodeset) is not None def node_is_gke(self, node_name=None) -> bool: - template_info = self.node_template_info(node_name) - return self.template_is_gke(template_info) + return self.nodeset_is_gke(self.node_nodeset(node_name)) def nodeset_is_gke(self, nodeset=None) -> bool: - template_info = self.template_info(nodeset.instance_template) - return self.template_is_gke(template_info) - - def template_is_gke(self, template_info=None) -> bool: - return "goog-gke-node" in template_info.labels + return "gke_nodepool" in nodeset def node_template(self, node_name=None) -> str: """ Self link of nodeset template """ diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf index dfcba87017..755b852299 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf @@ -34,7 +34,12 @@ output "slurm_bucket_path" { output "slurm_bucket_name" { description = "GCS Bucket name of Slurm cluster file storage." - value = module.slurm_files.bucket_name + value = module.bucket[0].name +} + +output "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + value = module.bucket } output "slurm_bucket_dir" { diff --git a/community/modules/scheduler/slinky/README.md b/community/modules/scheduler/slinky/README.md index 5f3eef0ced..bb9d2c72a6 100644 --- a/community/modules/scheduler/slinky/README.md +++ b/community/modules/scheduler/slinky/README.md @@ -167,7 +167,6 @@ No modules. | Name | Description | |------|-------------| -| [instructions](#output\_instructions) | Post deployment instructions. | | [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | | [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | diff --git a/community/modules/scheduler/slinky/outputs.tf b/community/modules/scheduler/slinky/outputs.tf index 9fd2882716..8ea6385905 100644 --- a/community/modules/scheduler/slinky/outputs.tf +++ b/community/modules/scheduler/slinky/outputs.tf @@ -12,22 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -output "instructions" { - description = "Post deployment instructions." - value = <<-EOT - To test Slurm functionality, connect to the controller to use Slurm client commands: - kubectl exec -it statefulsets/slurm-controller \ - --namespace=slurm \ - -- bash --login - - On the controller pod (e.g. host slurm@slurm-controller-0), run the following commands to quickly test Slurm is functioning: - sinfo - srun hostname - sbatch --wrap="sleep 60" - squeue - EOT -} - output "slurm_namespace" { description = "namespace for the slurm chart" value = var.slurm_namespace diff --git a/modules/README.md b/modules/README.md index 37831fc708..19b37b5b45 100644 --- a/modules/README.md +++ b/modules/README.md @@ -54,6 +54,8 @@ Modules that are still in development and less stable are labeled with the * **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. * **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. +* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. [vm-instance]: compute/vm-instance/README.md [gke-node-pool]: ../modules/compute/gke-node-pool/README.md diff --git a/modules/compute/gke-node-pool/README.md b/modules/compute/gke-node-pool/README.md index f3ad67d0d4..e740c1b64a 100644 --- a/modules/compute/gke-node-pool/README.md +++ b/modules/compute/gke-node-pool/README.md @@ -376,7 +376,7 @@ limitations under the License. | [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | | [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | | [machine\_type](#output\_machine\_type) | Machine Type | -| [node\_count](#output\_node\_count) | The number of nodes in the node pool. | +| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | | [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | | [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | | [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | diff --git a/modules/compute/gke-node-pool/outputs.tf b/modules/compute/gke-node-pool/outputs.tf index e5be0f9608..3fa92c6953 100644 --- a/modules/compute/gke-node-pool/outputs.tf +++ b/modules/compute/gke-node-pool/outputs.tf @@ -111,8 +111,8 @@ output "instructions" { value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null } -output "node_count" { - description = "The number of nodes in the node pool." +output "node_count_static" { + description = "The number of static nodes in node-pool." value = coalesce(var.static_node_count, var.initial_node_count, 0) } diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml b/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml index 631efbeef7..94db240ed3 100644 --- a/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml @@ -189,6 +189,8 @@ hosts: remote_host gather_facts: false # must wait until host is reachable ignore_unreachable: true # ensure always block will run even if SSH fails + vars: + key_type: "munge" tasks: - name: Slurm Test Block vars: @@ -202,6 +204,13 @@ ansible.builtin.wait_for: path: /var/run/munge/munge.socket.2 timeout: 600 + when: key_type == 'munge' + + - name: Wait until Slurm key exists + ansible.builtin.wait_for: + path: /etc/slurm/slurm.key + timeout: 600 # Waits for up to 10 minutes for the file to appear + when: key_type == 'slurm' - name: Count Slurm nodes ansible.builtin.shell: diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml b/tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml index 29280d638e..c5d9a431d0 100644 --- a/tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml @@ -38,6 +38,10 @@ changed_when: gcluster_destroy.changed run_once: true ansible.builtin.command: ./gcluster destroy {{ deployment_name }} --auto-approve + # Temporarily add retry due to K8S deletion issue. + until: gcluster_destroy.rc == 0 + retries: 1 + delay: 10 args: chdir: "{{ workspace }}" environment: diff --git a/tools/cloud-build/daily-tests/builds/slurm-gke.yaml b/tools/cloud-build/daily-tests/builds/slurm-gke.yaml new file mode 100644 index 0000000000..1df3fd5a08 --- /dev/null +++ b/tools/cloud-build/daily-tests/builds/slurm-gke.yaml @@ -0,0 +1,56 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +tags: +- m.filestore +- m.private-service-access +- m.schedmd-slurm-gcp-v6-controller +- m.vpc +- m.gke-cluster +- m.gke-node-pool +- m.gke-nodeset +- m.gke-partition +- m.kubectl-apply +- m.service-account +- m.schedmd-slurm-gcp-v6-login +- m.schedmd-slurm-gcp-v6-nodeset +- m.schedmd-slurm-gcp-v6-partition +- m.slinky +- slurm6 + +timeout: 14400s # 4hr +steps: +# While using static network names we are gaurding against more than 1 instance running at a time (for multi-group tests) +- id: check_for_running_build + name: gcr.io/cloud-builders/gcloud + script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/slurm-gke.yaml" + +- id: slurm-gke + name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + cd /workspace && make + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:3} + + ansible-playbook -v tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-gke.yml" diff --git a/tools/cloud-build/daily-tests/tests/slurm-gke.yml b/tools/cloud-build/daily-tests/tests/slurm-gke.yml new file mode 100644 index 0000000000..2db5099197 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/slurm-gke.yml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +test_name: slurm-gke +deployment_name: slurm{{ build }} +slurm_cluster_name: "slurm{{ build[0:4] }}" +zone: us-west4-c + +key_type: slurm + +cli_deployment_vars: + network_name: "{{ network }}" + region: us-west4 + zone: us-west4-c + +workspace: /workspace +blueprint_yaml: "{{ workspace }}/community/examples/slurm-gke/slurm-gke.yaml" +network: "{{ test_name }}-net" +# Note: Pattern matching in gcloud only supports 1 wildcard, a*-login-* won't work. +login_node: "{{ slurm_cluster_name }}-slurm-login-*" +controller_node: "{{ slurm_cluster_name }}-controller" +post_deploy_tests: +- test-validation/test-mounts.yml +- test-validation/test-partitions.yml +custom_vars: + partitions: + - gke + mounts: + - /home