Skip to content

Commit 5a169be

Browse files
bghirabghira
authored
Merge pull request #1819 from bghira/main
merge 3.0.2 to release branch
2 parents 7bbbc0f + f6d2f4f commit 5a169be

File tree

687 files changed

+150161
-65459
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

687 files changed

+150161
-65459
lines changed

‎.flake8‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[flake8]
2+
max-line-length = 125
3+
exclude =
4+
.venv,
5+
__pycache__,
6+
.git,
7+
build,
8+
dist,
9+
.eggs,
10+
*.egg-info

‎.github/workflows/publish-pypi.yaml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: Publish to PyPI
2+
3+
on:
4+
release:
5+
types: [published]
6+
workflow_dispatch:
7+
inputs:
8+
test_pypi:
9+
description: 'Publish to Test PyPI instead of PyPI'
10+
required: false
11+
default: false
12+
type: boolean
13+
14+
jobs:
15+
build-wheels:
16+
runs-on: ubuntu-latest
17+
strategy:
18+
matrix:
19+
python-version: ["3.11", "3.12"]
20+
21+
steps:
22+
- uses: actions/checkout@v4
23+
24+
- name: Set up Python ${{ matrix.python-version }}
25+
uses: actions/setup-python@v4
26+
with:
27+
python-version: ${{ matrix.python-version }}
28+
29+
- name: Install build dependencies
30+
run: |
31+
python -m pip install --upgrade pip
32+
pip install build wheel setuptools
33+
34+
- name: Build universal wheel
35+
run: |
36+
python -m build --wheel
37+
env:
38+
SIMPLETUNER_PLATFORM: cpu
39+
40+
- name: Upload wheel artifacts
41+
uses: actions/upload-artifact@v4
42+
with:
43+
name: wheels-py${{ matrix.python-version }}
44+
path: dist/*.whl
45+
46+
build-sdist:
47+
runs-on: ubuntu-latest
48+
steps:
49+
- uses: actions/checkout@v4
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v4
53+
with:
54+
python-version: "3.11"
55+
56+
- name: Install build dependencies
57+
run: |
58+
python -m pip install --upgrade pip
59+
pip install build
60+
61+
- name: Build source distribution
62+
run: python -m build --sdist
63+
64+
- name: Upload sdist artifacts
65+
uses: actions/upload-artifact@v4
66+
with:
67+
name: sdist
68+
path: dist/*.tar.gz
69+
70+
publish:
71+
needs: [build-wheels, build-sdist]
72+
runs-on: ubuntu-latest
73+
environment: release
74+
permissions:
75+
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
76+
77+
steps:
78+
- name: Download all artifacts
79+
uses: actions/download-artifact@v4
80+
with:
81+
path: dist/
82+
merge-multiple: true
83+
84+
- name: List artifacts
85+
run: ls -la dist/
86+
87+
- name: Publish to Test PyPI
88+
if: ${{ github.event.inputs.test_pypi == 'true' }}
89+
uses: pypa/gh-action-pypi-publish@release/v1
90+
with:
91+
repository-url: https://test.pypi.org/legacy/
92+
password: ${{ secrets.PYPI_API_TOKEN }}
93+
skip-existing: true
94+
95+
- name: Publish to PyPI
96+
if: ${{ github.event.inputs.test_pypi != 'true' }}
97+
uses: pypa/gh-action-pypi-publish@release/v1
98+
with:
99+
password: ${{ secrets.PYPI_API_TOKEN }}
100+
skip-existing: true

‎.github/workflows/python-tests.yaml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,8 @@ jobs:
2525
with:
2626
python-version: 3.11
2727

28-
- name: Install Poetry
29-
run: python -m pip install --upgrade pip poetry
30-
3128
- name: Install Dependencies
32-
run: poetry -C install/apple install
29+
run: python -m pip install --upgrade pip && pip install -e .[test]
3330

3431
- name: Run Tests
35-
run: poetry -C ./ -P install/apple run python -m unittest discover tests/
32+
run: python -m unittest discover tests/

‎.gitignore‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,19 @@ webhooks.json
6262
untracked/
6363
config/*/
6464
config/*.toml
65-
static
66-
templates
6765
api_state.json
6866
images/
6967
datasets/
7068
.test_runner_state
7169
.test_runner_state.json
70+
.coverage
71+
coverage.xml
72+
*/htmlcov
73+
.simpletuner_output
74+
.mypy_cache
75+
.ruff_cache
76+
test-screenshots/
77+
test-folder/
78+
node_modules
79+
package-lock.json
80+
**/config

‎.pre-commit-config.yaml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
repos:
2+
# Pre-commit hooks for basic file hygiene
3+
- repo: https://github.com/pre-commit/pre-commit-hooks
4+
rev: v4.6.0
5+
hooks:
6+
- id: trailing-whitespace
7+
- id: end-of-file-fixer
8+
- id: check-yaml
9+
- id: check-added-large-files
10+
- id: check-case-conflict
11+
- id: check-merge-conflict
12+
- id: debug-statements
13+
- id: mixed-line-ending
14+
args: ['--fix=lf']
15+
16+
# Python code formatting with Black
17+
- repo: https://github.com/psf/black
18+
rev: 24.8.0
19+
hooks:
20+
- id: black
21+
language_version: python3
22+
args: ['--line-length=125']
23+
24+
# Import sorting with isort
25+
- repo: https://github.com/pycqa/isort
26+
rev: 5.13.2
27+
hooks:
28+
- id: isort
29+
args: ['--profile=black', '--line-length=125']
30+
31+
# Python linting with flake8 (using local venv, enforcing 100-char limit)
32+
- repo: local
33+
hooks:
34+
- id: flake8
35+
name: flake8
36+
entry: .venv/bin/python
37+
args: ['-m', 'flake8', '--max-line-length=125', '--extend-ignore=E501,E266,E203,W503,F841,F401,E402,F811,E722,E721,F541']
38+
language: system
39+
types: [python]
40+
exclude: '^(scripts/|tests/|\.venv/)'
41+
42+
# Type checking with mypy (disabled - too many issues for now)
43+
# - repo: local
44+
# hooks:
45+
# - id: mypy
46+
# name: mypy
47+
# entry: .venv/bin/python
48+
# args: ['-m', 'mypy', '--ignore-missing-imports', '--no-strict-optional']
49+
# language: system
50+
# types: [python]
51+
# exclude: '^(scripts/|tests/)'

‎Dockerfile‎

Lines changed: 59 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,82 @@
11
# SimpleTuner needs CU141
22
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
33

4-
# /workspace is the default volume for Runpod & other hosts
5-
WORKDIR /workspace
6-
7-
# Update apt-get
8-
RUN apt-get update -y
4+
ARG PYTHON_VERSION=3.11
95

10-
# Prevents different commands from being stuck by waiting
11-
# on user input during build
12-
ENV DEBIAN_FRONTEND noninteractive
6+
# Prevent commands from blocking for input during build
7+
ENV DEBIAN_FRONTEND=noninteractive
138

14-
# Install libg dependencies
15-
RUN apt install libgl1-mesa-glx -y
16-
RUN apt-get install 'ffmpeg'\
17-
'libsm6'\
18-
'libxext6' -y
19-
20-
# Install misc unix libraries
21-
RUN apt-get install -y --no-install-recommends openssh-server \
22-
openssh-client \
23-
git \
24-
git-lfs \
25-
wget \
26-
curl \
27-
tmux \
28-
tldr \
29-
nvtop \
30-
vim \
31-
rsync \
32-
net-tools \
33-
less \
34-
iputils-ping \
35-
7zip \
36-
zip \
37-
unzip \
38-
htop \
39-
inotify-tools
9+
# /workspace is the default volume for Runpod & other hosts
10+
WORKDIR /workspace
4011

41-
# Set up git to support LFS, and to store credentials; useful for Huggingface Hub
12+
# Base system dependencies (including Python ${PYTHON_VERSION} toolchain)
13+
RUN apt-get update -y && \
14+
apt-get install -y --no-install-recommends \
15+
build-essential \
16+
ca-certificates \
17+
curl \
18+
ffmpeg \
19+
git \
20+
git-lfs \
21+
htop \
22+
inotify-tools \
23+
iputils-ping \
24+
less \
25+
libgl1-mesa-glx \
26+
libsm6 \
27+
libxext6 \
28+
libopenmpi-dev \
29+
net-tools \
30+
nvtop \
31+
openmpi-bin \
32+
openssh-client \
33+
openssh-server \
34+
p7zip-full \
35+
python${PYTHON_VERSION} \
36+
python${PYTHON_VERSION}-dev \
37+
python${PYTHON_VERSION}-venv \
38+
rsync \
39+
tmux \
40+
tldr \
41+
unzip \
42+
vim \
43+
wget \
44+
zip && \
45+
rm -rf /var/lib/apt/lists/*
46+
47+
# Configure git to support LFS and credential storage
4248
RUN git config --global credential.helper store && \
4349
git lfs install
4450

45-
# Install Python VENV
46-
RUN apt-get install -y python3.10-venv
51+
# Create a dedicated virtual environment with the requested Python version
52+
RUN python${PYTHON_VERSION} -m venv /opt/venv && \
53+
/opt/venv/bin/pip install --upgrade pip setuptools wheel
54+
55+
# Use the virtual environment for all subsequent Python work
56+
ENV VIRTUAL_ENV=/opt/venv
57+
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
4758

4859
# Ensure SSH access. Not needed for Runpod but is required on Vast and other Docker hosts
4960
EXPOSE 22/tcp
5061

51-
# Python
52-
RUN apt-get update -y && apt-get install -y python3 python3-pip
53-
RUN python3 -m pip install pip --upgrade
54-
55-
# HF
62+
# HuggingFace cache location and platform hint for setup.py
5663
ENV HF_HOME=/workspace/huggingface
64+
ENV SIMPLETUNER_PLATFORM=cuda
5765

58-
RUN pip3 install "huggingface_hub[cli]"
59-
60-
# WanDB
61-
RUN pip3 install wandb
66+
# Install supporting CLIs ahead of the project install
67+
RUN pip install --no-cache-dir "huggingface_hub[cli]" wandb
6268

63-
# Clone SimpleTuner
64-
RUN git clone https://github.com/bghira/SimpleTuner --branch release
65-
# RUN git clone https://github.com/bghira/SimpleTuner --branch main # Uncomment to use latest (possibly unstable) version
69+
# Install MPI bindings needed for CUDA multi-node support
70+
RUN pip install --no-cache-dir mpi4py
6671

67-
# Install SimpleTuner
68-
RUN pip3 install poetry
69-
RUN cd SimpleTuner && python3 -m venv .venv && poetry install --no-root
70-
RUN chmod +x SimpleTuner/train.sh
72+
# Install SimpleTuner from PyPI to match published releases
73+
RUN pip install --no-cache-dir simpletuner
7174

7275
# Copy start script with exec permissions
7376
COPY --chmod=755 docker-start.sh /start.sh
7477

78+
# Ensure we remain in the default workspace location
79+
WORKDIR /workspace
80+
7581
# Dummy entrypoint
7682
ENTRYPOINT [ "/start.sh" ]

0 commit comments

Comments
 (0)