Merge pull request PaddlePaddle#51 from JunnYu/ppdiffusers_v0.16.3

westfish · web-flow · commit efd6cff98d98 · 2023-08-10T15:28:07.000+08:00
[PPDiffusers] Release PPDiffusers v0.16.3
diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,7 @@ __pycache__/
 *.so
 
 # json file
-*.json
+# *.json
 
 # log file
 *.log
diff --git a/ppdiffusers/VERSION b/ppdiffusers/VERSION
@@ -1 +1 @@
-0.16.1
+0.16.3
diff --git a/ppdiffusers/deploy/controlnet/infer_dygraph.py b/ppdiffusers/deploy/controlnet/infer_dygraph.py
@@ -198,6 +198,12 @@ def main(args):
                     continue
                 else:
                     raise ValueError(e)
+
+        if not args.use_fp16 and attention_type == "flash":
+            print(
+                "Flash attention is not supported dtype=float32! Please use float16 or bfloat16. We will skip this!"
+            )
+            continue
         guess_mode = args.guess_mode
         width = args.width
         height = args.height
diff --git a/ppdiffusers/deploy/controlnet/infer_dygraph_torch.py b/ppdiffusers/deploy/controlnet/infer_dygraph_torch.py
@@ -126,13 +126,13 @@ def parse_arguments():
         "--pretrained_model_name_or_path",
         type=str,
         default="runwayml/stable-diffusion-v1-5",
-        help="Path to the `ppdiffusers` checkpoint to convert (either a local directory or on the bos).",
+        help="Path to the `diffusers` checkpoint to convert (either a local directory or on the bos).",
     )
     parser.add_argument(
         "--controlnet_pretrained_model_name_or_path",
         type=str,
         default="lllyasviel/sd-controlnet-canny",
-        help="Path to the `ppdiffusers` controlnet_pretrained_model_name_or_path  checkpoint to convert (either a local directory or on the bos).",
+        help="Path to the `diffusers` controlnet_pretrained_model_name_or_path  checkpoint to convert (either a local directory or on the bos).",
     )
     parser.add_argument(
         "--inference_steps",
diff --git a/ppdiffusers/deploy/infer_dygraph.py b/ppdiffusers/deploy/infer_dygraph.py
@@ -161,6 +161,11 @@ def main(args):
                 else:
                     raise ValueError(e)
 
+        if not args.use_fp16 and attention_type == "flash":
+            print(
+                "Flash attention is not supported dtype=float32! Please use float16 or bfloat16. We will skip this!"
+            )
+            continue
         width = args.width
         height = args.height
         hr_resize_width = args.hr_resize_width
diff --git a/ppdiffusers/deploy/requirements.txt b/ppdiffusers/deploy/requirements.txt
@@ -1 +1 @@
-ppdiffusers>=0.16.1
+ppdiffusers>=0.16.3
diff --git a/ppdiffusers/deploy/scripts/test_controlnet_infer_dygraph_torch.sh b/ppdiffusers/deploy/scripts/test_controlnet_infer_dygraph_torch.sh
@@ -14,17 +14,17 @@
 cd ../controlnet
 pip install diffusers==0.17.1
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --controlnet_pretrained_model_name_or_path lllyasviel/sd-controlnet-canny \
     --task_name all --use_fp16 True \
     --attention_type raw --benchmark_steps 10 --device_id 0 --parse_prompt_type raw
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --controlnet_pretrained_model_name_or_path lllyasviel/sd-controlnet-canny \
     --task_name all --use_fp16 True \
     --attention_type sdp --benchmark_steps 10 --device_id 0 --parse_prompt_type raw
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --controlnet_pretrained_model_name_or_path lllyasviel/sd-controlnet-canny \
     --task_name all --use_fp16 True \
     --attention_type sdp --benchmark_steps 10 --device_id 0 --parse_prompt_type raw --compile True --channels_last True
diff --git a/ppdiffusers/deploy/scripts/test_infer_dygraph_torch.sh b/ppdiffusers/deploy/scripts/test_infer_dygraph_torch.sh
@@ -14,15 +14,15 @@
 cd ..
 pip install diffusers==0.17.1
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --task_name all --use_fp16 True \
     --attention_type raw --benchmark_steps 10 --device_id 0 --parse_prompt_type raw
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --task_name all --use_fp16 True \
     --attention_type sdp --benchmark_steps 10 --device_id 0 --parse_prompt_type raw
 
-python infer_dygraph_toch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
+python infer_dygraph_torch.py --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \
     --task_name all --use_fp16 True \
     --attention_type sdp --benchmark_steps 10 --device_id 0 --parse_prompt_type raw --compile True --channels_last True
 
diff --git a/ppdiffusers/examples/autoencoder/vae/config/vae.json b/ppdiffusers/examples/autoencoder/vae/config/vae.json
@@ -0,0 +1,36 @@
+{
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_out_channels": null,
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "in_channels": 3,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 512,
+  "up_block_out_channels": [
+    128,
+    256,
+    256,
+    256,
+    512
+  ],
+  "latent_channels": 4,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}
diff --git a/ppdiffusers/examples/t2i-adapter/config/openpose_adapter.json b/ppdiffusers/examples/t2i-adapter/config/openpose_adapter.json
@@ -0,0 +1,16 @@
+{
+  "_class_name": "Adapter",
+  "_ppdiffusers_version": "0.2.2",
+  "channels_in": 3,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "num_res_blocks": 2,
+  "kernel_size": 1,
+  "res_block_skip": true,
+  "use_conv": false,
+  "input_scale_factor": 8
+}
diff --git a/ppdiffusers/examples/text_to_image_laion400m/config/ldmbert.json b/ppdiffusers/examples/text_to_image_laion400m/config/ldmbert.json
@@ -0,0 +1,15 @@
+{
+    "vocab_size": 30522,
+    "max_position_embeddings": 77,
+    "encoder_layers": 32,
+    "encoder_ffn_dim": 5120,
+    "encoder_attention_heads": 8,
+    "head_dim": 64,
+    "activation_function": "gelu",
+    "d_model": 1280,
+    "dropout": 0.0,
+    "attention_dropout": 0.0,
+    "activation_dropout": 0.0,
+    "init_std": 0.02,
+    "pad_token_id": 0
+}
diff --git a/ppdiffusers/examples/text_to_image_laion400m/config/unet.json b/ppdiffusers/examples/text_to_image_laion400m/config/unet.json
@@ -0,0 +1,35 @@
+{
+    "act_fn": "silu",
+    "attention_head_dim": 8,
+    "block_out_channels": [
+      320,
+      640,
+      1280,
+      1280
+    ],
+    "center_input_sample": false,
+    "cross_attention_dim": 1280,
+    "down_block_types": [
+      "CrossAttnDownBlock2D",
+      "CrossAttnDownBlock2D",
+      "CrossAttnDownBlock2D",
+      "DownBlock2D"
+    ],
+    "downsample_padding": 1,
+    "flip_sin_to_cos": true,
+    "freq_shift": 0,
+    "in_channels": 4,
+    "layers_per_block": 2,
+    "mid_block_scale_factor": 1,
+    "norm_eps": 1e-05,
+    "norm_num_groups": 32,
+    "out_channels": 4,
+    "sample_size": 32,
+    "up_block_types": [
+      "UpBlock2D",
+      "CrossAttnUpBlock2D",
+      "CrossAttnUpBlock2D",
+      "CrossAttnUpBlock2D"
+    ]
+  }
+  
diff --git a/ppdiffusers/ppdiffusers/models/autoencoder_kl.py b/ppdiffusers/ppdiffusers/models/autoencoder_kl.py
@@ -166,7 +166,7 @@ def disable_slicing(self):
     def encode(self, x: paddle.Tensor,
                return_dict: bool=True) -> AutoencoderKLOutput:
         # TODO junnyu, support float16
-        x = x.cast(self.dtype)
+        x = x.cast(self.encoder.conv_in.weight.dtype)
         if self.use_tiling and (x.shape[-1] > self.tile_sample_min_size or
                                 x.shape[-2] > self.tile_sample_min_size):
             return self.tiled_encode(x, return_dict=return_dict)
@@ -198,7 +198,7 @@ def _decode(self, z: paddle.Tensor,
     def decode(self, z: paddle.Tensor,
                return_dict: bool=True) -> Union[DecoderOutput, paddle.Tensor]:
         # TODO junnyu, add this to support pure fp16
-        z = z.cast(self.dtype)
+        z = z.cast(self.post_quant_conv.weight.dtype)
         if self.use_slicing and z.shape[0] > 1:
             # split、chunk paddle vs pytorch may have some difference
             decoded_slices = [
@@ -339,7 +339,7 @@ def forward(
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`DecoderOutput`] instead of a plain tuple.
         """
-        x = sample.cast(self.dtype)
+        x = sample
         posterior = self.encode(x).latent_dist
         if sample_posterior:
             z = posterior.sample(generator=generator)
diff --git a/ppdiffusers/ppdiffusers/models/vae.py b/ppdiffusers/ppdiffusers/models/vae.py
@@ -226,7 +226,7 @@ def forward(self, z):
         sample = z
         sample = self.conv_in(sample)
 
-        upscale_dtype = self.dtype
+        upscale_dtype = self.up_blocks.dtype
         if self.training and self.gradient_checkpointing and not sample.stop_gradient:
 
             def create_custom_forward(module):
diff --git a/ppdiffusers/ppdiffusers/patches/ppnlp_patch_utils.py b/ppdiffusers/ppdiffusers/patches/ppnlp_patch_utils.py
@@ -390,7 +390,9 @@ def scaled_dot_product_attention_(
             if attention_op in [None, "auto"]:
                 head_dim = query.shape[-1]
                 attention_op = "cutlass"
-                if is_support_flash_attention:
+                if is_support_flash_attention and query.dtype in [
+                        paddle.float16, paddle.bfloat16
+                ]:
                     if flash_attn_version == 1:
                         if head_dim <= 128:
                             attention_op = "flash"
diff --git a/ppdiffusers/ppdiffusers/pipelines/fastdeploy_utils.py b/ppdiffusers/ppdiffusers/pipelines/fastdeploy_utils.py
@@ -537,11 +537,12 @@ def unet_hidden_states_dim(self):
 
     @property
     def text_encoder_hidden_states_dim(self):
-        if not hasattr(self, "text_encoder") or self.text_encoder is None:
+        if self.text_encoder is None:
             return 768
-        return self.text_encoder.model.get_output_info(0).shape[2]
+        dim = self.text_encoder.model.get_output_info(0).shape[2]
+        return 768 if dim == -1 else dim
 
-    def change_scheduler(self, scheduler_type="ddim"):
+    def change_scheduler(self, scheduler_type="ddim", inplace=True):
         scheduler_type = scheduler_type.lower()
         if scheduler_type == "pndm":
             scheduler = PNDMScheduler.from_config(
@@ -595,7 +596,9 @@ def change_scheduler(self, scheduler_type="ddim"):
             raise ValueError(
                 f"Scheduler of type {scheduler_type} doesn't exist! Please choose in {self.supported_scheduler}!"
             )
-        self.scheduler = scheduler
+        if inplace:
+            self.scheduler = scheduler
+        return scheduler
 
     def get_timesteps(self, num_inference_steps, strength=1.0):
         if strength >= 1:
@@ -883,7 +886,7 @@ def _encode_prompt(
                 prompt_embeds=prompt_embeds,
                 negative_prompt_embeds=negative_prompt_embeds,
                 max_embeddings_multiples=max_embeddings_multiples,
-                infer_op=infer_op,
+                infer_op="raw",  # NOTE: we can't use zero copy!
                 **kwargs, )
         elif parse_prompt_type == "raw":
             return self._encode_prompt_raw(
@@ -958,7 +961,7 @@ def _encode_prompt_lpw(
                 prompt=prompt,
                 uncond_prompt=uncond_tokens,
                 max_embeddings_multiples=max_embeddings_multiples,
-                infer_op="raw",  # NOTE: we can't use zero copy!
+                infer_op=infer_op,
                 **kwargs, )
 
         bs_embed, seq_len, _ = prompt_embeds.shape
@@ -1218,6 +1221,8 @@ def __call__(self, **kwargs):
 
         inputs = {}
         for k, v in kwargs.items():
+            if v is None:
+                continue
             if k == "timestep":
                 v = v.astype("float32")
             inputs[k] = v
diff --git a/ppdiffusers/ppdiffusers/utils/dynamic_modules_utils.py b/ppdiffusers/ppdiffusers/utils/dynamic_modules_utils.py
@@ -151,6 +151,10 @@ def check_imports(filename):
         try:
             importlib.import_module(imp)
         except ImportError:
+            if imp == "ligo":
+                imp = "ligo-segments"
+            if imp == "cv2":
+                imp = "opencv-python"
             missing_packages.append(imp)
 
     if len(missing_packages) > 0:
diff --git a/ppdiffusers/ppdiffusers/version.py b/ppdiffusers/ppdiffusers/version.py
@@ -14,4 +14,4 @@
 
 # this file will be generated by tools
 # please not modify it.
-VERSION = "0.16.1"
+VERSION = "0.16.3"
diff --git a/ppdiffusers/requirements.txt b/ppdiffusers/requirements.txt
@@ -3,3 +3,4 @@ safetensors
 ftfy
 regex
 Pillow
+einops>=0.6.1
diff --git a/ppdiffusers/tests/examples_test/download_sd15.py b/ppdiffusers/tests/examples_test/download_sd15.py
@@ -0,0 +1,3 @@
+from ppdiffusers import DiffusionPipeline
+# CompVis/stable-diffusion-v1-4
+p = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
diff --git a/ppdiffusers/tests/examples_test/fire.sh b/ppdiffusers/tests/examples_test/fire.sh
@@ -0,0 +1 @@
+nohup sh train.sh 1> train.log 2>&1 &
diff --git a/ppdiffusers/tests/examples_test/prepare.sh b/ppdiffusers/tests/examples_test/prepare.sh
@@ -0,0 +1,44 @@
+pip install -r requirements.txt
+pip uninstall ppdiffusers -y
+cd ../../
+pip install -e .
+cd -
+
+python download_sd15.py
+
+cd ../../examples/dreambooth
+wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/dogs.tar.gz
+tar -zxvf dogs.tar.gz
+cd -
+
+cd ../../examples/textual_inversion
+wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/cat-toy.tar.gz
+tar -zxvf cat-toy.tar.gz
+cd -
+
+cd ../../examples/text_to_image_laion400m
+wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/laion400m_demo_data.tar.gz
+rm -rf data
+tar -zxvf laion400m_demo_data.tar.gz
+cp laion400m_demo_data.tar.gz ../stable_diffusion
+cp laion400m_demo_data.tar.gz ../autoencoder/vae
+cd -
+
+cd ../../examples/stable_diffusion
+rm -rf data
+tar -zxvf laion400m_demo_data.tar.gz
+cd -
+
+cd ../../examples/autoencoder/vae
+rm -rf data
+tar -zxvf laion400m_demo_data.tar.gz
+cd -
+
+cd ../../examples/controlnet
+wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/fill50k.zip
+unzip -o fill50k.zip
+
+cd -
+cd ../../examples/t2i-adapter
+wget https://paddlenlp.bj.bcebos.com/models/community/westfish/t2i-adapter/openpose_data_demo.tar.gz
+tar -zxvf openpose_data_demo.tar.gz
diff --git a/ppdiffusers/tests/examples_test/requirements.txt b/ppdiffusers/tests/examples_test/requirements.txt
@@ -0,0 +1,17 @@
+paddlenlp>=2.6.0rc0
+ppdiffusers>=0.16.1
+scipy
+safetensors
+pyyaml
+opencv-python
+albumentations
+fastcore
+visualdl
+requests
+Pillow
+cchardet
+datasets
+visualdl
+einops
+ligo-segments
+opencv-python-headless<=4.3
diff --git a/ppdiffusers/tests/examples_test/train.sh b/ppdiffusers/tests/examples_test/train.sh

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-ppdiffusers>=0.16.1`
	`1`	`+ppdiffusers>=0.16.3`