Skip to content

Commit d95094c

Browse files
joey12300joey12300
authored
[Diffusion] Add C++ dpm solver (#714)
* Add BetaForAlphaBar, ConvertModelOutput, SetTimesteps, and constructor for DPMSolverMultistepScheduler * tmp * Add DPMSolverFirstOrderUpdate * Add ScaleModelInput * Add MultiStepDPMSolverSecondOrderUpdate * add MultiStepDPMSolverThirdOrderUpdate * Add Step * Add FASTDEPLOY_DECL * Add AddNoise * Fix operator * update * Fix DPMSolverMultistepScheduler * Upgrade Slice * Fix DPMSolverFirstOrderUpdate * remove FASTDEPLOY_DECL * Add config for dpm solver
1 parent 3f8ed9b commit d95094c

File tree

14 files changed

+675
-11
lines changed

14 files changed

+675
-11
lines changed

‎examples/multimodal/stable_diffusion/cpp/CMakeLists.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
PROJECT(main C CXX)
16+
CMAKE_MINIMUM_REQUIRED (VERSION 3.10)
17+
18+
option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
19+
set(THIRD_LIBS "")
20+
include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
21+
22+
include_directories(${FASTDEPLOY_INCS})
23+
24+
file(GLOB_RECURSE ALL_SRCS ${PROJECT_SOURCE_DIR}/*.cc)
25+
26+
add_executable(main ${ALL_SRCS})
27+
target_link_libraries(main ${FASTDEPLOY_LIBS} ${THIRD_LIBS})

‎examples/multimodal/stable_diffusion/cpp/dpm_solver_multistep_scheduler.cc‎

Lines changed: 395 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/multimodal/stable_diffusion/cpp/dpm_solver_multistep_scheduler.h‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#pragma once
16+
17+
#include "./scheduler.h"
18+
#include "fastdeploy/core/fd_tensor.h"
19+
20+
namespace fastdeploy {
21+
22+
class DPMSolverMultistepScheduler : public Scheduler {
23+
public:
24+
DPMSolverMultistepScheduler(int num_train_timesteps = 1000,
25+
float beta_start = 0.0001, float beta_end = 0.02,
26+
const std::string& beta_schedule = "linear",
27+
const std::vector<float>& trained_betas = {},
28+
int solver_order = 2, bool predict_epsilon = true,
29+
bool thresholding = false,
30+
float dynamic_thresholding_ratio = 0.995,
31+
float sample_max_value = 1.0,
32+
const std::string& algorithm_type = "dpmsolver++",
33+
const std::string& solver_type = "midpoint",
34+
bool lower_order_final = true);
35+
void BetaForAlphaBar(FDTensor* out, int num_diffusion_timesteps,
36+
float max_beta = 0.999);
37+
void ConvertModelOutput(const FDTensor& model_output, int timestep,
38+
const FDTensor& sample, FDTensor* out);
39+
void DPMSolverFirstOrderUpdate(const FDTensor& model_output, int timestep,
40+
int prev_timestep, const FDTensor& sample,
41+
FDTensor* out);
42+
void MultiStepDPMSolverSecondOrderUpdate(
43+
const std::vector<FDTensor>& model_output_list,
44+
const std::vector<int>& timestep_list, int prev_timestep,
45+
const FDTensor& sample, FDTensor* out);
46+
void MultiStepDPMSolverThirdOrderUpdate(
47+
const std::vector<FDTensor>& model_output_list,
48+
const std::vector<int>& timestep_list, int prev_timestep,
49+
const FDTensor& sample, FDTensor* out);
50+
void SetTimesteps(int num_inference_steps) override;
51+
void Step(const FDTensor& model_output, int timestep, const FDTensor& sample,
52+
FDTensor* prev_sample) override;
53+
void ScaleModelInput(const FDTensor& sample, FDTensor* out,
54+
const std::vector<FDTensor>& timesteps = {}) override;
55+
void AddNoise(const FDTensor& original_samples, const FDTensor& noise,
56+
const FDTensor& timesteps, FDTensor* out) override;
57+
struct Config {
58+
int num_train_timesteps_;
59+
float beta_start_;
60+
float beta_end_;
61+
std::string beta_schedule_;
62+
int solver_order_;
63+
bool predict_epsilon_;
64+
bool thresholding_;
65+
float dynamic_thresholding_ratio_;
66+
float sample_max_value_;
67+
std::string algorithm_type_;
68+
std::string solver_type_;
69+
bool lower_order_final_;
70+
} config;
71+
72+
private:
73+
FDTensor betas_;
74+
FDTensor alphas_;
75+
FDTensor alphas_cumprod_;
76+
FDTensor alpha_t_;
77+
FDTensor sigma_t_;
78+
FDTensor lambda_t_;
79+
int num_inference_steps_;
80+
FDTensor timesteps_;
81+
int lower_order_nums_;
82+
std::vector<FDTensor> model_outputs_;
83+
};
84+
85+
} // namespace fastdeploy

‎examples/multimodal/stable_diffusion/cpp/main.cc‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "dpm_solver_multistep_scheduler.h"
16+
#include <iostream>
17+
18+
int main() {
19+
fastdeploy::DPMSolverMultistepScheduler dpm(
20+
/* num_train_timesteps */ 1000,
21+
/* beta_start = */ 0.00085,
22+
/* beta_end = */ 0.012,
23+
/* beta_schedule = */ "scaled_linear",
24+
/* trained_betas = */ {},
25+
/* solver_order = */ 2,
26+
/* predict_epsilon = */ true,
27+
/* thresholding = */ false,
28+
/* dynamic_thresholding_ratio = */ 0.995,
29+
/* sample_max_value = */ 1.0,
30+
/* algorithm_type = */ "dpmsolver++",
31+
/* solver_type = */ "midpoint",
32+
/* lower_order_final = */ true);
33+
34+
return 0;
35+
}

‎examples/multimodal/stable_diffusion/cpp/scheduler.h‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#pragma once
16+
17+
#include "fastdeploy/core/fd_tensor.h"
18+
19+
namespace fastdeploy {
20+
21+
class Scheduler {
22+
virtual void SetTimesteps(int num_inference_steps) = 0;
23+
virtual void Step(const FDTensor& model_output, int timestep,
24+
const FDTensor& sample, FDTensor* prev_sample) = 0;
25+
virtual void ScaleModelInput(const FDTensor& sample, FDTensor* out,
26+
const std::vector<FDTensor>& timesteps = {}) = 0;
27+
virtual void AddNoise(const FDTensor& original_samples, const FDTensor& noise,
28+
const FDTensor& timesteps, FDTensor* out) = 0;
29+
};
30+
31+
} // namespace fastdeploy

‎fastdeploy/core/fd_tensor.cc‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414
#include "fastdeploy/core/fd_tensor.h"
15-
#include "fastdeploy/core/fd_scalar.h"
1615
#include "fastdeploy/core/float16.h"
1716
#include "fastdeploy/utils/utils.h"
1817

@@ -81,8 +80,7 @@ const void* FDTensor::CpuData() const {
8180

8281
void FDTensor::SetExternalData(const std::vector<int64_t>& new_shape,
8382
const FDDataType& data_type, void* data_buffer,
84-
const Device& new_device,
85-
int new_device_id) {
83+
const Device& new_device, int new_device_id) {
8684
dtype = data_type;
8785
shape.assign(new_shape.begin(), new_shape.end());
8886
external_data_ptr = data_buffer;

‎fastdeploy/core/fd_tensor.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,11 @@
1919
#include <vector>
2020

2121
#include "fastdeploy/core/allocate.h"
22+
#include "fastdeploy/core/fd_scalar.h"
2223
#include "fastdeploy/core/fd_type.h"
2324

2425
namespace fastdeploy {
2526

26-
struct Scalar;
27-
2827
struct FASTDEPLOY_DECL FDTensor {
2928
// std::vector<int8_t> data;
3029
void* buffer_ = nullptr;

‎fastdeploy/function/clip.cc‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,15 @@ void ClipKernel(const FDTensor& x, double min, double max, FDTensor* out) {
3939
"max should be greater than or equal to min. But received min = %f, "
4040
"max = %f",
4141
static_cast<float>(min_), static_cast<float>(max_));
42-
43-
out->Allocate(x.Shape(), x.Dtype());
42+
FDTensor tmp;
43+
tmp.Allocate(x.Shape(), x.Dtype());
4444
const T* x_data = reinterpret_cast<const T*>(x.Data());
4545

4646
int64_t numel = x.Numel();
47-
T* out_data = reinterpret_cast<T*>(out->Data());
47+
T* out_data = reinterpret_cast<T*>(tmp.Data());
4848

4949
std::transform(x_data, x_data + numel, out_data, ClipFunctor<T>(min_, max_));
50+
*out = std::move(tmp);
5051
}
5152

5253
void Clip(const FDTensor& x, double min, double max, FDTensor* out) {

‎fastdeploy/function/elementwise.cc‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,25 @@ FDTensor operator/(const FDTensor& x, const FDTensor& y) {
8686
return out;
8787
}
8888

89+
#define INSTANTIATE_OPERATOR(operation_type) \
90+
template FDTensor operator operation_type(const FDTensor& x, bool y); \
91+
template FDTensor operator operation_type(const FDTensor& x, uint8_t y); \
92+
template FDTensor operator operation_type(const FDTensor& x, int16_t y); \
93+
template FDTensor operator operation_type(const FDTensor& x, int y); \
94+
template FDTensor operator operation_type(const FDTensor& x, int64_t y); \
95+
template FDTensor operator operation_type(const FDTensor& x, float y); \
96+
template FDTensor operator operation_type(const FDTensor& x, double y); \
97+
template FDTensor operator operation_type(bool x, const FDTensor& y); \
98+
template FDTensor operator operation_type(uint8_t x, const FDTensor& y); \
99+
template FDTensor operator operation_type(int16_t x, const FDTensor& y); \
100+
template FDTensor operator operation_type(int x, const FDTensor& y); \
101+
template FDTensor operator operation_type(int64_t x, const FDTensor& y); \
102+
template FDTensor operator operation_type(float x, const FDTensor& y); \
103+
template FDTensor operator operation_type(double x, const FDTensor& y)
104+
105+
INSTANTIATE_OPERATOR(+);
106+
INSTANTIATE_OPERATOR(-);
107+
INSTANTIATE_OPERATOR(*);
108+
INSTANTIATE_OPERATOR(/);
109+
89110
} // namespace fastdeploy

‎fastdeploy/function/elementwise.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414

1515
#pragma once
1616

17+
#include "fastdeploy/core/fd_scalar.h"
1718
#include "fastdeploy/core/fd_tensor.h"
1819

1920
namespace fastdeploy {
21+
2022
namespace function {
2123

2224
/** Excute the add operation for input FDTensors. *out = x + y.
@@ -62,10 +64,42 @@ FASTDEPLOY_DECL void Maximum(const FDTensor& x, const FDTensor& y,
6264

6365
FASTDEPLOY_DECL FDTensor operator+(const FDTensor& x, const FDTensor& y);
6466

67+
template <typename T> FDTensor operator+(const FDTensor& x, T y) {
68+
return x + FDTensor(Scalar(y));
69+
}
70+
71+
template <typename T> FDTensor operator+(T x, const FDTensor& y) {
72+
return FDTensor(Scalar(x)) + y;
73+
}
74+
6575
FASTDEPLOY_DECL FDTensor operator-(const FDTensor& x, const FDTensor& y);
6676

77+
template <typename T> FDTensor operator-(const FDTensor& x, T y) {
78+
return x - FDTensor(Scalar(y));
79+
}
80+
81+
template <typename T> FDTensor operator-(T x, const FDTensor& y) {
82+
return FDTensor(Scalar(x)) - y;
83+
}
84+
6785
FASTDEPLOY_DECL FDTensor operator*(const FDTensor& x, const FDTensor& y);
6886

87+
template <typename T> FDTensor operator*(const FDTensor& x, T y) {
88+
return x * FDTensor(Scalar(y));
89+
}
90+
91+
template <typename T> FDTensor operator*(T x, const FDTensor& y) {
92+
return FDTensor(Scalar(x)) * y;
93+
}
94+
6995
FASTDEPLOY_DECL FDTensor operator/(const FDTensor& x, const FDTensor& y);
7096

97+
template <typename T> FDTensor operator/(const FDTensor& x, T y) {
98+
return x / FDTensor(Scalar(y));
99+
}
100+
101+
template <typename T> FDTensor operator/(T x, const FDTensor& y) {
102+
return FDTensor(Scalar(x)) / y;
103+
}
104+
71105
} // namespace fastdeploy

0 commit comments

Comments
 (0)