Skip to content

Sd3 5 #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DarthAffe merged 3 commits into from
Jan 3, 2025
Merged

Sd3 5 #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f79b74e
Added flash attention and slg layer skip parameters; added comments f…
DarthAffe Jan 3, 2025
b174c2a
Updated backend in example
DarthAffe Jan 3, 2025
b7e5ee7
Added flash attention parameter to example
DarthAffe Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Examples/ImageCreationUI/ImageCreationUI.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@

<ItemGroup>
<PackageReference Include="HPPH.System.Drawing" Version="1.0.0" />
<PackageReference Include="StableDiffusion.NET.Backend.Cpu" Version="3.2.0" />
<PackageReference Include="StableDiffusion.NET.Backend.Cuda" Version="3.2.0" />
<PackageReference Include="StableDiffusion.NET.Backend.Rocm" Version="3.2.0" />
<PackageReference Include="StableDiffusion.NET.Backend.Cpu" Version="3.3.1" />
<PackageReference Include="StableDiffusion.NET.Backend.Cuda" Version="3.3.1" />
<PackageReference Include="StableDiffusion.NET.Backend.Rocm" Version="3.3.1" />
</ItemGroup>

<ItemGroup>
Expand Down
5 changes: 5 additions & 0 deletions Examples/ImageCreationUI/MainWindow.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@
<Label Content="Schedule" />
<ComboBox ItemsSource="{Binding Source={StaticResource ScheduleDataSource}}" SelectedItem="{Binding Schedule}" />

<StackPanel Orientation="Horizontal" Margin="0,4,0,0">
<Label Content="Flash Attention" />
<CheckBox VerticalAlignment="Center" IsChecked="{Binding FlashAttention}" />
</StackPanel>

<Button Margin="0,8" Content="Load Model" Command="{Binding LoadModelCommand}" IsEnabled="{Binding IsReady}" />

<Separator />
Expand Down
11 changes: 9 additions & 2 deletions Examples/ImageCreationUI/MainWindowViewModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@ public Schedule Schedule
set => SetProperty(ref _schedule, value);
}

private bool _flashAttention = true;
public bool FlashAttention
{
get => _flashAttention;
set => SetProperty(ref _flashAttention, value);
}

private string _prompt = string.Empty;
public string Prompt
{
Expand Down Expand Up @@ -242,14 +249,14 @@ private async void LoadModel()
restoreDefaultParameters = _model?.ModelParameter.DiffusionModelType != DiffusionModelType.StableDiffusion;

LogLine($"Loading stable diffusion-model '{ModelPath}'");
_model = await Task.Run(() => ModelBuilder.StableDiffusion(ModelPath).WithMultithreading().WithVae(VaePath).WithSchedule(Schedule).Build());
_model = await Task.Run(() => ModelBuilder.StableDiffusion(ModelPath).WithMultithreading().WithVae(VaePath).WithSchedule(Schedule).WithFlashAttention(FlashAttention).Build());
}
else if (IsFluxSelected)
{
restoreDefaultParameters = _model?.ModelParameter.DiffusionModelType != DiffusionModelType.Flux;

LogLine($"Loading flux-model '{DiffusionModelPath}'");
_model = await Task.Run(() => ModelBuilder.Flux(DiffusionModelPath, ClipLPath, T5xxlPath, VaePath).WithMultithreading().WithSchedule(Schedule).Build());
_model = await Task.Run(() => ModelBuilder.Flux(DiffusionModelPath, ClipLPath, T5xxlPath, VaePath).WithMultithreading().WithSchedule(Schedule).WithFlashAttention(FlashAttention).Build());
}
else
{
Expand Down
2 changes: 2 additions & 0 deletions StableDiffusion.NET/Enums/Quantization.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ public enum Quantization
Q4_0_4_4 = 31,
Q4_0_4_8 = 32,
Q4_0_8_8 = 33,
TQ1_0 = 34,
TQ2_0 = 35,

Unspecified
}
8 changes: 8 additions & 0 deletions StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,4 +119,12 @@ public static T WithSchedule<T>(this T builder, Schedule schedule)

return builder;
}

public static T WithFlashAttention<T>(this T builder, bool flashAttention = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.FlashAttention = flashAttention;

return builder;
}
}
45 changes: 38 additions & 7 deletions StableDiffusion.NET/Models/DiffusionModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ private void Initialize()
ModelParameter.Schedule,
ModelParameter.KeepClipOnCPU,
ModelParameter.KeepControlNetOnCPU,
ModelParameter.KeepVaeOnCPU);
ModelParameter.KeepVaeOnCPU,
ModelParameter.FlashAttention);

if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model.");
}
Expand Down Expand Up @@ -122,7 +123,12 @@ public IImage<ColorRGB> TextToImage(string prompt, DiffusionParameter? parameter
parameter.ControlNet.Strength,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);

Marshal.FreeHGlobal((nint)nativeControlNetImage.data);
}
Expand Down Expand Up @@ -152,7 +158,12 @@ public IImage<ColorRGB> TextToImage(string prompt, DiffusionParameter? parameter
parameter.ControlNet.Strength,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);
}
}
}
Expand All @@ -174,7 +185,12 @@ public IImage<ColorRGB> TextToImage(string prompt, DiffusionParameter? parameter
0,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);
}

return ImageHelper.ToImage(result);
Expand Down Expand Up @@ -246,7 +262,12 @@ private IImage<ColorRGB> ImageToImage(string prompt, Native.sd_image_t image, Di
parameter.ControlNet.Strength,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);

Marshal.FreeHGlobal((nint)nativeControlNetImage.data);
}
Expand Down Expand Up @@ -278,7 +299,12 @@ private IImage<ColorRGB> ImageToImage(string prompt, Native.sd_image_t image, Di
parameter.ControlNet.Strength,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);
}
}
}
Expand All @@ -302,7 +328,12 @@ private IImage<ColorRGB> ImageToImage(string prompt, Native.sd_image_t image, Di
0,
parameter.PhotoMaker.StyleRatio,
parameter.PhotoMaker.NormalizeInput,
parameter.PhotoMaker.InputIdImageDirectory);
parameter.PhotoMaker.InputIdImageDirectory,
parameter.SkipLayers,
parameter.SkipLayers.Length,
parameter.SlgScale,
parameter.SkipLayerStart,
parameter.SkipLayerEnd);
}

return ImageHelper.ToImage(result);
Expand Down
32 changes: 32 additions & 0 deletions StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,44 @@ public sealed class ControlNetParameter
{
public bool IsEnabled => Image != null;

/// <summary>
/// image condition, control net
/// </summary>
public IImage? Image { get; set; } = null;

/// <summary>
/// strength to apply Control Net (default: 0.9)
/// 1.0 corresponds to full destruction of information in init image
/// </summary>
public float Strength { get; set; } = 0.9f;

/// <summary>
/// apply canny preprocessor (edge detection)
/// </summary>
public bool CannyPreprocess { get; set; } = false;

/// <summary>
///
/// </summary>
public float CannyHighThreshold { get; set; } = 0.08f;

/// <summary>
///
/// </summary>
public float CannyLowThreshold { get; set; } = 0.08f;

/// <summary>
///
/// </summary>
public float CannyWeak { get; set; } = 0.8f;

/// <summary>
///
/// </summary>
public float CannyStrong { get; set; } = 1.0f;

/// <summary>
///
/// </summary>
public bool CannyInverse { get; set; } = false;
}
80 changes: 79 additions & 1 deletion StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,114 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz
{
public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None;

/// <summary>
/// path to vae
/// </summary>
public string VaePath { get; set; } = string.Empty;

/// <summary>
/// path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)
/// </summary>
public string TaesdPath { get; set; } = string.Empty;

/// <summary>
/// lora model directory
/// </summary>
public string LoraModelDirectory { get; set; } = string.Empty;

/// <summary>
/// path to embeddings
/// </summary>
public string EmbeddingsDirectory { get; set; } = string.Empty;

/// <summary>
/// path to control net model
/// </summary>
public string ControlNetPath { get; set; } = string.Empty;

/// <summary>
/// number of threads to use during computation (default: -1)
/// If threads = -1, then threads will be set to the number of CPU physical cores
/// </summary>
public int ThreadCount { get; set; } = 1;

/// <summary>
///
/// </summary>
public bool VaeDecodeOnly { get; set; } = false;

/// <summary>
/// process vae in tiles to reduce memory usage
/// </summary>
public bool VaeTiling { get; set; } = false;

/// <summary>
/// keep controlnet in cpu
/// </summary>
public bool KeepControlNetOnCPU { get; set; } = false;

/// <summary>
/// keep clip in cpu (for low vram)
/// </summary>
public bool KeepClipOnCPU { get; set; } = false;

/// <summary>
/// keep vae in cpu (for low vram)
/// </summary>
public bool KeepVaeOnCPU { get; set; } = false;

/// <summary>
/// use flash attention in the diffusion model (for low vram)
/// Might lower quality, since it implies converting k and v to f16.
/// This might crash if it is not supported by the backend.
/// </summary>
public bool FlashAttention { get; set; } = false;

/// <summary>
/// RNG (default: Standard)
/// </summary>
public RngType RngType { get; set; } = RngType.Standard;

/// <summary>
/// Denoiser sigma schedule (default: Default)
/// </summary>
public Schedule Schedule { get; set; } = Schedule.Default;

/// <summary>
///
/// </summary>
public Quantization Quantization { get; set; } = Quantization.Unspecified;

// SD <= 3 only
/// <summary>
/// path to full model
/// </summary>
public string ModelPath { get; set; } = string.Empty;

/// <summary>
/// path to PHOTOMAKER stacked id embeddings
/// </summary>
public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty;

// Flux & SD3.5 only
/// <summary>
/// path to the standalone diffusion model
/// </summary>
public string DiffusionModelPath { get; set; } = string.Empty;

/// <summary>
/// path to the clip-l text encoder
/// </summary>
public string ClipLPath { get; set; } = string.Empty;
public string T5xxlPath { get; set; } = string.Empty;

/// <summary>
/// path to the the t5xxl text encoder
/// </summary>
public string T5xxlPath { get; set; } = string.Empty;

// SD3.5 only
/// <summary>
/// path to the clip-g text encoder
/// </summary>
public string ClipGPath { get; set; } = string.Empty;
}
Loading