From c87062cbaa48f03a0782e38890f94ce1676f7be4 Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Sat, 1 Oct 2022 18:04:47 -0400 Subject: [PATCH 1/6] Support optional flag to clamp gradient in 'backward' to prevent crash This commit addresses a very intermittent, but deadly crash bug that is destroying my training runs - a very occasional infinite gradient in the 'backward' function. In this commit, functionality remains unchanged by default. However, an optional flag has been added that allows clamping the gradient in the 'backward' function. The flag takes the form of an int or sequence giving the max value (or min/max if sequence). An optional third value in the passed sequence is interpreted as a Boolean that indicates whether to print a warning to the console whenever an infinite gradient is clamped. The default is False. Support for PyTorch only. --- apps/gaussian_blur.py | 3 ++- apps/generative_models/mnist_vae.py | 3 ++- apps/generative_models/rendering.py | 3 ++- apps/render_svg.py | 3 ++- apps/seam_carving.py | 3 ++- apps/svg_brush.py | 3 ++- apps/texture_synthesis.py | 3 ++- pydiffvg/optimize_svg.py | 3 ++- pydiffvg/render_pytorch.py | 24 ++++++++++++++++++++++++ 9 files changed, 40 insertions(+), 8 deletions(-) diff --git a/apps/gaussian_blur.py b/apps/gaussian_blur.py index 8d148026..637534cd 100644 --- a/apps/gaussian_blur.py +++ b/apps/gaussian_blur.py @@ -6,7 +6,7 @@ import scipy.ndimage.filters as F -def render(canvas_width, canvas_height, shapes, shape_groups): +def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) @@ -16,6 +16,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups): 2, # num_samples_y 0, # seed None, + backward_clamp_gradient, *scene_args) return img diff --git a/apps/generative_models/mnist_vae.py b/apps/generative_models/mnist_vae.py index c0da6265..28ce6910 100644 --- a/apps/generative_models/mnist_vae.py +++ b/apps/generative_models/mnist_vae.py @@ -50,13 +50,14 @@ def _onehot(label): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene( - canvas_width, canvas_height, shapes, shape_groups) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) img = _render(canvas_width, canvas_height, samples, samples, 0, None, + backward_clamp_gradient, *scene_args) return img diff --git a/apps/generative_models/rendering.py b/apps/generative_models/rendering.py index 4ef475ec..80c8621e 100644 --- a/apps/generative_models/rendering.py +++ b/apps/generative_models/rendering.py @@ -12,7 +12,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, - seed=None): + seed=None, backward_clamp_gradient=None): if seed is None: seed = random.randint(0, 1000000) _render = pydiffvg.RenderFunction.apply @@ -21,6 +21,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, img = _render(canvas_width, canvas_height, samples, samples, seed, # seed None, # background image + backward_clamp_gradient, *scene_args) return img diff --git a/apps/render_svg.py b/apps/render_svg.py index 0aa92736..0090c5ef 100644 --- a/apps/render_svg.py +++ b/apps/render_svg.py @@ -7,7 +7,7 @@ import torch as th -def render(canvas_width, canvas_height, shapes, shape_groups): +def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) @@ -17,6 +17,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups): 2, # num_samples_y 0, # seed None, + backward_clamp_gradient, *scene_args) return img diff --git a/apps/seam_carving.py b/apps/seam_carving.py index aa0176ef..dfbaad65 100644 --- a/apps/seam_carving.py +++ b/apps/seam_carving.py @@ -109,7 +109,7 @@ def carve_seam(im): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) img = _render(canvas_width, # width canvas_height, # height @@ -117,6 +117,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): samples, # num_samples_y 0, # seed None, + backward_clamp_gradient, *scene_args) return img diff --git a/apps/svg_brush.py b/apps/svg_brush.py index de54e48e..adb50c33 100644 --- a/apps/svg_brush.py +++ b/apps/svg_brush.py @@ -35,7 +35,7 @@ def checkerboard(shape, square_size=2): res=bin*np.array([[[1., 1., 1.,]]])+(1-bin)*np.array([[[.75, .75, .75,]]]) return torch.tensor(res,requires_grad=False,dtype=torch.float32) -def render(optim, viewport): +def render(optim, viewport, backward_clamp_gradient=None): scene_args = pydiffvg.RenderFunction.serialize_scene(*optim.build_scene()) render = pydiffvg.RenderFunction.apply img = render(viewport[0], # width @@ -44,6 +44,7 @@ def render(optim, viewport): 2, # num_samples_y 0, # seed None, + backward_clamp_gradient, *scene_args) return img diff --git a/apps/texture_synthesis.py b/apps/texture_synthesis.py index 3a7ccce0..732a3abf 100644 --- a/apps/texture_synthesis.py +++ b/apps/texture_synthesis.py @@ -36,13 +36,14 @@ def texture_syn(img_path): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) img = _render(canvas_width, # width canvas_height, # height samples, # num_samples_x samples, # num_samples_y 0, # seed None, + backward_clamp_gradient, *scene_args) return img diff --git a/pydiffvg/optimize_svg.py b/pydiffvg/optimize_svg.py index ce0097f5..37db8979 100644 --- a/pydiffvg/optimize_svg.py +++ b/pydiffvg/optimize_svg.py @@ -1017,7 +1017,7 @@ def zero_grad(self): if issubclass(item.__class__,OptimizableSvg.SvgNode): item.zero_grad() - def render(self,scale=None,seed=0): + def render(self,scale=None,seed=0,backward_clamp_gradient=None): #render at native resolution scene = self.build_scene() scene_args = pydiffvg.RenderFunction.serialize_scene(*scene) @@ -1029,6 +1029,7 @@ def render(self,scale=None,seed=0): 2, # num_samples_y seed, # seed None, # background_image + backward_clamp_gradient, *scene_args) return img diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index b776ce67..27ccff90 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -179,6 +179,7 @@ def forward(ctx, num_samples_y, seed, background_image, + backward_clamp_gradient, *args): """ Forward rendering pass. @@ -427,6 +428,7 @@ def forward(ctx, ctx.output_type = output_type ctx.use_prefiltering = use_prefiltering ctx.eval_positions = eval_positions + ctx.backward_clamp_gradient = backward_clamp_gradient return rendered_image @staticmethod @@ -684,6 +686,28 @@ def backward(ctx, use_prefiltering = ctx.use_prefiltering eval_positions = ctx.eval_positions background_image = ctx.background_image + backward_clamp_gradient = ctx.backward_clamp_gradient + + if backward_clamp_gradient is None: + assert torch.isfinite(grad_img).all() + else: + try: + assert torch.isfinite(grad_img).all() + except: + if type(backward_clamp_gradient) is int: + backward_clamp_gradient_ = {"max": backward_clamp_gradient} + elif len(backward_clamp_gradient) == 1: + backward_clamp_gradient_ = {"max": backward_clamp_gradient[0]} + elif len(backward_clamp_gradient) >= 2: + backward_clamp_gradient_ = { + "min": backward_clamp_gradient[0], + "max": backward_clamp_gradient[1], + } + if len(backward_clamp_gradient) >= 3 and backward_clamp_gradient[2]: + print( + f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to {backward_clamp_gradient_}' + ) + grad_img = torch.clamp(grad_img, **backward_clamp_gradient_) if background_image is not None: d_background_image = torch.zeros_like(background_image) From e76089d1df249d37684742d05492cfd02b497295 Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Sat, 1 Oct 2022 20:02:55 -0400 Subject: [PATCH 2/6] Clarify use of backward_clamp_gradient_mag --- apps/gaussian_blur.py | 4 +-- apps/generative_models/mnist_vae.py | 4 +-- apps/generative_models/rendering.py | 4 +-- apps/render_svg.py | 4 +-- apps/seam_carving.py | 4 +-- apps/svg_brush.py | 4 +-- apps/texture_synthesis.py | 4 +-- pydiffvg/optimize_svg.py | 4 +-- pydiffvg/render_pytorch.py | 39 +++++++++++++++++------------ 9 files changed, 39 insertions(+), 32 deletions(-) diff --git a/apps/gaussian_blur.py b/apps/gaussian_blur.py index 637534cd..bf27c815 100644 --- a/apps/gaussian_blur.py +++ b/apps/gaussian_blur.py @@ -6,7 +6,7 @@ import scipy.ndimage.filters as F -def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None): +def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) @@ -16,7 +16,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gra 2, # num_samples_y 0, # seed None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/generative_models/mnist_vae.py b/apps/generative_models/mnist_vae.py index 28ce6910..f1a60ec1 100644 --- a/apps/generative_models/mnist_vae.py +++ b/apps/generative_models/mnist_vae.py @@ -50,14 +50,14 @@ def _onehot(label): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene( - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) img = _render(canvas_width, canvas_height, samples, samples, 0, None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/generative_models/rendering.py b/apps/generative_models/rendering.py index 80c8621e..9459c952 100644 --- a/apps/generative_models/rendering.py +++ b/apps/generative_models/rendering.py @@ -12,7 +12,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, - seed=None, backward_clamp_gradient=None): + seed=None, backward_clamp_gradient_mag=None): if seed is None: seed = random.randint(0, 1000000) _render = pydiffvg.RenderFunction.apply @@ -21,7 +21,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, img = _render(canvas_width, canvas_height, samples, samples, seed, # seed None, # background image - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/render_svg.py b/apps/render_svg.py index 0090c5ef..4374898e 100644 --- a/apps/render_svg.py +++ b/apps/render_svg.py @@ -7,7 +7,7 @@ import torch as th -def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None): +def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) @@ -17,7 +17,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gra 2, # num_samples_y 0, # seed None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/seam_carving.py b/apps/seam_carving.py index dfbaad65..d563eff1 100644 --- a/apps/seam_carving.py +++ b/apps/seam_carving.py @@ -109,7 +109,7 @@ def carve_seam(im): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) img = _render(canvas_width, # width canvas_height, # height @@ -117,7 +117,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): samples, # num_samples_y 0, # seed None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/svg_brush.py b/apps/svg_brush.py index adb50c33..2c100163 100644 --- a/apps/svg_brush.py +++ b/apps/svg_brush.py @@ -35,7 +35,7 @@ def checkerboard(shape, square_size=2): res=bin*np.array([[[1., 1., 1.,]]])+(1-bin)*np.array([[[.75, .75, .75,]]]) return torch.tensor(res,requires_grad=False,dtype=torch.float32) -def render(optim, viewport, backward_clamp_gradient=None): +def render(optim, viewport, backward_clamp_gradient_mag=None): scene_args = pydiffvg.RenderFunction.serialize_scene(*optim.build_scene()) render = pydiffvg.RenderFunction.apply img = render(viewport[0], # width @@ -44,7 +44,7 @@ def render(optim, viewport, backward_clamp_gradient=None): 2, # num_samples_y 0, # seed None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/texture_synthesis.py b/apps/texture_synthesis.py index 732a3abf..08300642 100644 --- a/apps/texture_synthesis.py +++ b/apps/texture_synthesis.py @@ -36,14 +36,14 @@ def texture_syn(img_path): def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient=None) + canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) img = _render(canvas_width, # width canvas_height, # height samples, # num_samples_x samples, # num_samples_y 0, # seed None, - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/pydiffvg/optimize_svg.py b/pydiffvg/optimize_svg.py index 37db8979..0c46d2f1 100644 --- a/pydiffvg/optimize_svg.py +++ b/pydiffvg/optimize_svg.py @@ -1017,7 +1017,7 @@ def zero_grad(self): if issubclass(item.__class__,OptimizableSvg.SvgNode): item.zero_grad() - def render(self,scale=None,seed=0,backward_clamp_gradient=None): + def render(self,scale=None,seed=0,backward_clamp_gradient_mag=None): #render at native resolution scene = self.build_scene() scene_args = pydiffvg.RenderFunction.serialize_scene(*scene) @@ -1029,7 +1029,7 @@ def render(self,scale=None,seed=0,backward_clamp_gradient=None): 2, # num_samples_y seed, # seed None, # background_image - backward_clamp_gradient, + backward_clamp_gradient_mag, *scene_args) return img diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index 27ccff90..fd296d32 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -179,7 +179,7 @@ def forward(ctx, num_samples_y, seed, background_image, - backward_clamp_gradient, + backward_clamp_gradient_mag, *args): """ Forward rendering pass. @@ -428,7 +428,7 @@ def forward(ctx, ctx.output_type = output_type ctx.use_prefiltering = use_prefiltering ctx.eval_positions = eval_positions - ctx.backward_clamp_gradient = backward_clamp_gradient + ctx.backward_clamp_gradient_mag = backward_clamp_gradient_mag return rendered_image @staticmethod @@ -686,28 +686,35 @@ def backward(ctx, use_prefiltering = ctx.use_prefiltering eval_positions = ctx.eval_positions background_image = ctx.background_image - backward_clamp_gradient = ctx.backward_clamp_gradient + backward_clamp_gradient_mag = ctx.backward_clamp_gradient_mag - if backward_clamp_gradient is None: + if backward_clamp_gradient_mag is None: assert torch.isfinite(grad_img).all() else: try: assert torch.isfinite(grad_img).all() except: - if type(backward_clamp_gradient) is int: - backward_clamp_gradient_ = {"max": backward_clamp_gradient} - elif len(backward_clamp_gradient) == 1: - backward_clamp_gradient_ = {"max": backward_clamp_gradient[0]} - elif len(backward_clamp_gradient) >= 2: - backward_clamp_gradient_ = { - "min": backward_clamp_gradient[0], - "max": backward_clamp_gradient[1], - } - if len(backward_clamp_gradient) >= 3 and backward_clamp_gradient[2]: + # backward_clamp_gradient_mag can be: + # - A single float or int defining the magnitude of the clamp in both directions + # - A sequence of at least one or two floats or ints defining the magnitude + # of the clamp in the min (element 0) and max (element 1, or element 0 if only one element) direction + # To print a warning to the console when the gradient is not finite, pass a sequence of length 3. + # The third element is treated as a Boolean and if True, a warning is printed. + if type(backward_clamp_gradient_mag) is int or type(backward_clamp_gradient_mag) is float: + min_: -float(abs(backward_clamp_gradient_mag)) + max_: +float(abs(backward_clamp_gradient_mag)) + elif len(backward_clamp_gradient_mag) == 1: + min_: -float(abs(backward_clamp_gradient_mag[0])) + max_: +float(abs(backward_clamp_gradient_mag[0])) + elif len(backward_clamp_gradient_mag) >= 2: + min_: -float(abs(backward_clamp_gradient_mag[0])) + max_: +float(abs(backward_clamp_gradient_mag[1])) + backward_clamp_gradient_mag = {"min": min_, "max": max_} + if len(backward_clamp_gradient_mag) >= 3 and backward_clamp_gradient_mag[2]: print( - f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to {backward_clamp_gradient_}' + f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to {backward_clamp_gradient_mag}' ) - grad_img = torch.clamp(grad_img, **backward_clamp_gradient_) + grad_img = torch.clamp(grad_img, **backward_clamp_gradient_mag) if background_image is not None: d_background_image = torch.zeros_like(background_image) From e61d3af3394b2bd366b1ff97c21471937d9b1633 Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Sun, 2 Oct 2022 01:38:09 -0400 Subject: [PATCH 3/6] Correction in location of backward_clamp_gradient_mag in various functions Also, add dummy argument in 'backward' to match new backward_clamp_gradient_mag argument --- apps/gaussian_blur.py | 2 +- apps/generative_models/mnist_vae.py | 8 ++++---- apps/render_svg.py | 2 +- apps/seam_carving.py | 6 +++--- apps/svg_brush.py | 2 +- apps/texture_synthesis.py | 6 +++--- pydiffvg/render_pytorch.py | 1 + 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/apps/gaussian_blur.py b/apps/gaussian_blur.py index bf27c815..bdb9c9ae 100644 --- a/apps/gaussian_blur.py +++ b/apps/gaussian_blur.py @@ -15,7 +15,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gra 2, # num_samples_x 2, # num_samples_y 0, # seed - None, + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/generative_models/mnist_vae.py b/apps/generative_models/mnist_vae.py index f1a60ec1..e403a26f 100644 --- a/apps/generative_models/mnist_vae.py +++ b/apps/generative_models/mnist_vae.py @@ -47,16 +47,16 @@ def _onehot(label): return label_onehot.float() -def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): +def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, backward_clamp_gradient_mag=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene( - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) + canvas_width, canvas_height, shapes, shape_groups) img = _render(canvas_width, canvas_height, samples, samples, - 0, - None, + 0, # seed + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/render_svg.py b/apps/render_svg.py index 4374898e..cc237c48 100644 --- a/apps/render_svg.py +++ b/apps/render_svg.py @@ -16,7 +16,7 @@ def render(canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gra 2, # num_samples_x 2, # num_samples_y 0, # seed - None, + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/seam_carving.py b/apps/seam_carving.py index d563eff1..d75bd36e 100644 --- a/apps/seam_carving.py +++ b/apps/seam_carving.py @@ -106,17 +106,17 @@ def carve_seam(im): return new_im -def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): +def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, backward_clamp_gradient_mag=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) + canvas_width, canvas_height, shapes, shape_groups) img = _render(canvas_width, # width canvas_height, # height samples, # num_samples_x samples, # num_samples_y 0, # seed - None, + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/svg_brush.py b/apps/svg_brush.py index 2c100163..3dec0e3a 100644 --- a/apps/svg_brush.py +++ b/apps/svg_brush.py @@ -43,7 +43,7 @@ def render(optim, viewport, backward_clamp_gradient_mag=None): 2, # num_samples_x 2, # num_samples_y 0, # seed - None, + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/apps/texture_synthesis.py b/apps/texture_synthesis.py index 08300642..c837c0dc 100644 --- a/apps/texture_synthesis.py +++ b/apps/texture_synthesis.py @@ -33,16 +33,16 @@ def texture_syn(img_path): return np.array(target_img) -def render(canvas_width, canvas_height, shapes, shape_groups, samples=2): +def render(canvas_width, canvas_height, shapes, shape_groups, samples=2, backward_clamp_gradient_mag=None): _render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene(\ - canvas_width, canvas_height, shapes, shape_groups, backward_clamp_gradient_mag=None) + canvas_width, canvas_height, shapes, shape_groups) img = _render(canvas_width, # width canvas_height, # height samples, # num_samples_x samples, # num_samples_y 0, # seed - None, + None, # background_image backward_clamp_gradient_mag, *scene_args) return img diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index fd296d32..55b32851 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -750,6 +750,7 @@ def backward(ctx, d_args.append(None) # num_samples_y d_args.append(None) # seed d_args.append(d_background_image) + d_args.append(None) # backward_clamp_gradient_mag d_args.append(None) # canvas_width d_args.append(None) # canvas_height d_args.append(None) # num_shapes From 6f504f3983e011c967163ef52cb047eda96d30bf Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Sun, 2 Oct 2022 01:52:01 -0400 Subject: [PATCH 4/6] Correction in printing of optional warning message when gradient is clamped --- pydiffvg/render_pytorch.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index 55b32851..e3d6be80 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -81,7 +81,7 @@ def serialize_scene(canvas_width, else: args.append(torch.zeros(shape.points.shape[0] - 1, dtype = torch.int32)) args.append(shape.points.cpu()) - args.append(None) + args.append(None) args.append(shape.is_closed) args.append(False) # use_distance_approx elif isinstance(shape, pydiffvg.Rect): @@ -376,7 +376,7 @@ def forward(ctx, assert(eval_positions.shape[0] == 0) rendered_image = torch.zeros(height, width, 4, device = pydiffvg.get_device()) else: - assert(output_type == OutputType.sdf) + assert(output_type == OutputType.sdf) if eval_positions.shape[0] == 0: rendered_image = torch.zeros(height, width, 1, device = pydiffvg.get_device()) else: @@ -459,7 +459,7 @@ def render_grad(grad_img, use_prefiltering = args[current_index] current_index += 1 eval_positions = args[current_index] - current_index += 1 + current_index += 1 shapes = [] shape_groups = [] shape_contents = [] # Important to avoid GC deleting the shapes @@ -709,11 +709,11 @@ def backward(ctx, elif len(backward_clamp_gradient_mag) >= 2: min_: -float(abs(backward_clamp_gradient_mag[0])) max_: +float(abs(backward_clamp_gradient_mag[1])) + if len(backward_clamp_gradient_mag) >= 3 and backward_clamp_gradient_mag[2]: + print( + f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to: {min_}/{max_}') + ) backward_clamp_gradient_mag = {"min": min_, "max": max_} - if len(backward_clamp_gradient_mag) >= 3 and backward_clamp_gradient_mag[2]: - print( - f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to {backward_clamp_gradient_mag}' - ) grad_img = torch.clamp(grad_img, **backward_clamp_gradient_mag) if background_image is not None: From b2b00a9b60c5d4cf7693eed931b62bbd347c7d9b Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Sun, 2 Oct 2022 02:02:52 -0400 Subject: [PATCH 5/6] Close parens typo --- pydiffvg/render_pytorch.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index e3d6be80..29002570 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -710,9 +710,7 @@ def backward(ctx, min_: -float(abs(backward_clamp_gradient_mag[0])) max_: +float(abs(backward_clamp_gradient_mag[1])) if len(backward_clamp_gradient_mag) >= 3 and backward_clamp_gradient_mag[2]: - print( - f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to: {min_}/{max_}') - ) + print(f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to: {min_}/{max_}') backward_clamp_gradient_mag = {"min": min_, "max": max_} grad_img = torch.clamp(grad_img, **backward_clamp_gradient_mag) From b75965efcd206a9a2235e1ad2c9f95f37e65f85a Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Wed, 2 Nov 2022 10:34:37 -0400 Subject: [PATCH 6/6] Syntax corrections, and remove old test for finite gradient, since that's the whole purpose of this PR --- pydiffvg/render_pytorch.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pydiffvg/render_pytorch.py b/pydiffvg/render_pytorch.py index 29002570..63d34b38 100644 --- a/pydiffvg/render_pytorch.py +++ b/pydiffvg/render_pytorch.py @@ -674,7 +674,6 @@ def backward(ctx, grad_img): if not grad_img.is_contiguous(): grad_img = grad_img.contiguous() - assert(torch.isfinite(grad_img).all()) scene = ctx.scene width = ctx.width @@ -701,14 +700,14 @@ def backward(ctx, # To print a warning to the console when the gradient is not finite, pass a sequence of length 3. # The third element is treated as a Boolean and if True, a warning is printed. if type(backward_clamp_gradient_mag) is int or type(backward_clamp_gradient_mag) is float: - min_: -float(abs(backward_clamp_gradient_mag)) - max_: +float(abs(backward_clamp_gradient_mag)) + min_ = -float(abs(backward_clamp_gradient_mag)) + max_ = +float(abs(backward_clamp_gradient_mag)) elif len(backward_clamp_gradient_mag) == 1: - min_: -float(abs(backward_clamp_gradient_mag[0])) - max_: +float(abs(backward_clamp_gradient_mag[0])) + min_ = -float(abs(backward_clamp_gradient_mag[0])) + max_ = +float(abs(backward_clamp_gradient_mag[0])) elif len(backward_clamp_gradient_mag) >= 2: - min_: -float(abs(backward_clamp_gradient_mag[0])) - max_: +float(abs(backward_clamp_gradient_mag[1])) + min_ = -float(abs(backward_clamp_gradient_mag[0])) + max_ = +float(abs(backward_clamp_gradient_mag[1])) if len(backward_clamp_gradient_mag) >= 3 and backward_clamp_gradient_mag[2]: print(f'Pydiffvg::backward "isfinite" assertion failed: clamping gradient to: {min_}/{max_}') backward_clamp_gradient_mag = {"min": min_, "max": max_}