From 48083734238865b0c3355ba21a29da222512a357 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sat, 31 Aug 2024 22:00:31 +0200 Subject: [PATCH 1/7] Fix kappa sigma usage --- src/vstarstack/library/calibration/dark.py | 4 +-- src/vstarstack/library/calibration/flat.py | 31 ++++++---------------- src/vstarstack/library/merge/__init__.py | 17 ------------ src/vstarstack/tool/merge.py | 6 +++-- tests/test_merge.py | 13 ++++----- 5 files changed, 21 insertions(+), 50 deletions(-) diff --git a/src/vstarstack/library/calibration/dark.py b/src/vstarstack/library/calibration/dark.py index beb09c23..c9c06d2d 100644 --- a/src/vstarstack/library/calibration/dark.py +++ b/src/vstarstack/library/calibration/dark.py @@ -14,7 +14,7 @@ import vstarstack.library.common import vstarstack.library.data -import vstarstack.library.merge +import vstarstack.library.merge.simple_mean def remove_dark(dataframe : vstarstack.library.data.DataFrame, dark : vstarstack.library.data.DataFrame): @@ -33,4 +33,4 @@ def remove_dark(dataframe : vstarstack.library.data.DataFrame, def prepare_darks(images : vstarstack.library.common.IImageSource ) -> vstarstack.library.data.DataFrame: """Build dark frame""" - return vstarstack.library.merge.simple_mean(images) + return vstarstack.library.merge.simple_mean.mean(images) diff --git a/src/vstarstack/library/calibration/flat.py b/src/vstarstack/library/calibration/flat.py index 9fd73fde..be9dd5b3 100644 --- a/src/vstarstack/library/calibration/flat.py +++ b/src/vstarstack/library/calibration/flat.py @@ -22,6 +22,7 @@ import vstarstack.library.stars.detect import vstarstack.library.stars.cut import vstarstack.library.image_process.blur +import vstarstack.library.merge.kappa_sigma from vstarstack.library.image_process.blur import BlurredSource from vstarstack.library.image_process.normalize import normalize @@ -78,9 +79,8 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, smooth_size : int ) -> vstarstack.library.data.DataFrame: """Generate flat image""" - sum_layer = {} - sum_weight = {} params = {} + no_star_images = [] for dataframe in images.items(): descs = [] params = dataframe.params @@ -90,27 +90,12 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, continue channel_descs = vstarstack.library.stars.detect.detect_stars(layer) descs += channel_descs + dataframe = vstarstack.library.image_process.blur.blur(dataframe, 5) dataframe = normalize(dataframe) - nostars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs) - for name in nostars_dataframe.get_channels(): - layer, opts = nostars_dataframe.get_channel(name) - if not opts["brightness"]: - continue - weight_name = nostars_dataframe.links["weight"][name] - weight, _ = nostars_dataframe.get_channel(weight_name) - if name not in sum_layer: - sum_layer[name] = layer - sum_weight[name] = weight - else: - sum_layer[name] += layer - sum_weight[name] += weight - - flat = vstarstack.library.data.DataFrame(params=params) - for name, layer in sum_layer.items(): - weight = sum_weight[name] - layer[np.where(weight == 0)] = 0 - flat.add_channel(layer, name, brightness=True) - flat.add_channel(weight, "weight-"+name, weight=True) - flat.add_channel_link(name, "weight-"+name, "weight") + no_stars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs) + no_star_images.append(no_stars_dataframe) + + no_star_source = vstarstack.library.common.ListImageSource(no_star_images) + flat = vstarstack.library.merge.kappa_sigma.kappa_sigma(no_star_source, 1, 1, 2) return flat diff --git a/src/vstarstack/library/merge/__init__.py b/src/vstarstack/library/merge/__init__.py index 79d22d71..e69de29b 100644 --- a/src/vstarstack/library/merge/__init__.py +++ b/src/vstarstack/library/merge/__init__.py @@ -1,17 +0,0 @@ -# -# Copyright (c) 2023 Vladislav Tsendrovskii -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3 of the License. -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -from vstarstack.library.merge.kappa_sigma import kappa_sigma -from vstarstack.library.merge.simple_add import simple_add -from vstarstack.library.merge.simple_mean import mean as simple_mean diff --git a/src/vstarstack/tool/merge.py b/src/vstarstack/tool/merge.py index ce3b14f5..f72fc337 100644 --- a/src/vstarstack/tool/merge.py +++ b/src/vstarstack/tool/merge.py @@ -16,6 +16,8 @@ import vstarstack.library.data import vstarstack.library.merge +import vstarstack.library.merge.simple_add +import vstarstack.library.merge.kappa_sigma import vstarstack.tool.cfg import vstarstack.tool.usage import vstarstack.tool.common @@ -33,7 +35,7 @@ def simple_add(project: vstarstack.tool.cfg.Project, argv: list): imgs = vstarstack.tool.common.listfiles(path_images, ".zip") filenames = [img[1] for img in imgs] - dataframe = vstarstack.library.merge.simple_add(FilesImageSource(filenames)) + dataframe = vstarstack.library.merge.simple_add.simple_add(FilesImageSource(filenames)) if dataframe is not None: vstarstack.tool.common.check_dir_exists(out) dataframe.store(out) @@ -55,7 +57,7 @@ def sigma_clip(project: vstarstack.tool.cfg.Project, argv: list): imgs = vstarstack.tool.common.listfiles(path_images, ".zip") filenames = [img[1] for img in imgs] - dataframe = vstarstack.library.merge.kappa_sigma(FilesImageSource(filenames), + dataframe = vstarstack.library.merge.kappa_sigma.kappa_sigma(FilesImageSource(filenames), kappa1, kappa2, sigma_steps) diff --git a/tests/test_merge.py b/tests/test_merge.py index 92918cf7..ba35f234 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -16,7 +16,8 @@ import os from vstarstack.library.loaders.classic import readjpeg -import vstarstack.library.merge +import vstarstack.library.merge.kappa_sigma +import vstarstack.library.merge.simple_add from vstarstack.library.common import ListImageSource dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -32,7 +33,7 @@ def test_simple_add(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.simple_add(source) + summ = vstarstack.library.merge.simple_add.simple_add(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -95,7 +96,7 @@ def test_kappa_sigma_1(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 0) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 0) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -112,7 +113,7 @@ def test_kappa_sigma_2(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 1) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 1) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -127,7 +128,7 @@ def test_kappa_sigma_3(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 2) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 2) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -155,7 +156,7 @@ def test_kappa_sigma_4(): noised.append(copy) source = ListImageSource(noised) - summ = vstarstack.library.merge.kappa_sigma(source, 3, 3, 5) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 3, 3, 5) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] From ccc6a6d2b91feb009e446e9d3b8552e51667a387 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sat, 31 Aug 2024 23:25:16 +0200 Subject: [PATCH 2/7] Build flats --- src/vstarstack/library/calibration/flat.py | 14 ++++--- .../library/calibration/removehot.py | 37 +++++++++++++++++++ src/vstarstack/library/data.py | 20 ++++++++++ src/vstarstack/library/stars/cut.py | 2 +- 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 src/vstarstack/library/calibration/removehot.py diff --git a/src/vstarstack/library/calibration/flat.py b/src/vstarstack/library/calibration/flat.py index be9dd5b3..4a45668e 100644 --- a/src/vstarstack/library/calibration/flat.py +++ b/src/vstarstack/library/calibration/flat.py @@ -25,8 +25,8 @@ import vstarstack.library.merge.kappa_sigma from vstarstack.library.image_process.blur import BlurredSource -from vstarstack.library.image_process.normalize import normalize from vstarstack.library.image_process.nanmean_filter import nanmean_filter +from vstarstack.library.calibration.removehot import remove_hot_pixels def flatten(dataframe : vstarstack.library.data.DataFrame, flat : vstarstack.library.data.DataFrame): @@ -79,11 +79,9 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, smooth_size : int ) -> vstarstack.library.data.DataFrame: """Generate flat image""" - params = {} no_star_images = [] for dataframe in images.items(): descs = [] - params = dataframe.params for name in dataframe.get_channels(): layer, opts = dataframe.get_channel(name) if not opts["brightness"]: @@ -91,11 +89,17 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, channel_descs = vstarstack.library.stars.detect.detect_stars(layer) descs += channel_descs - dataframe = vstarstack.library.image_process.blur.blur(dataframe, 5) - dataframe = normalize(dataframe) no_stars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs) + no_stars_dataframe = remove_hot_pixels(no_stars_dataframe) no_star_images.append(no_stars_dataframe) no_star_source = vstarstack.library.common.ListImageSource(no_star_images) flat = vstarstack.library.merge.kappa_sigma.kappa_sigma(no_star_source, 1, 1, 2) + for channel in flat.get_channels(): + layer, opts = flat.get_channel(channel) + if not flat.get_channel_option(channel, "signal"): + continue + layer = cv2.GaussianBlur(layer, (15, 15), 0) + flat.add_channel(layer, channel, **opts) + return flat diff --git a/src/vstarstack/library/calibration/removehot.py b/src/vstarstack/library/calibration/removehot.py new file mode 100644 index 00000000..7a139e5a --- /dev/null +++ b/src/vstarstack/library/calibration/removehot.py @@ -0,0 +1,37 @@ +"""Remove hot pixels""" +# +# Copyright (c) 2024 Vladislav Tsendrovskii +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import numpy as np +from vstarstack.library.data import DataFrame + +def remove_hot_pixels(image : DataFrame) -> DataFrame: + """ + Remove hot pixels from image + + + """ + for channel in image.get_channels(): + layer, opts = image.get_channel(channel) + if not image.get_channel_option(channel, "signal"): + continue + hotpixels = np.where(layer >= np.median(layer)*3) + layer[hotpixels] = 0 + image.add_channel(layer, channel, **opts) + + weight, _, weight_channel = image.get_linked_channel(channel, "weight") + if weight_channel is not None: + weight[hotpixels] = 0 + image.add_channel(weight, weight_channel, weight=True) + return image diff --git a/src/vstarstack/library/data.py b/src/vstarstack/library/data.py index 975f764d..e9aad2f0 100644 --- a/src/vstarstack/library/data.py +++ b/src/vstarstack/library/data.py @@ -178,6 +178,26 @@ def get_channel_option(self, channel : str, option : str) -> bool | None: return False return self.channels[channel]["options"][option] + def get_linked_channel(self, channel : str, link_type : str): + """ + Get linked channel + + Parameters: + channel (str) - channel name + link_type (str) - type of link + + Returns: + None, None, None if no such link or no linked channel + layer, opts, name if such linked channel exists + """ + if link_type not in self.links: + return None, None, None + if channel not in self.links[link_type]: + return None, None, None + name = self.links[link_type][channel] + layer, opts = self.get_channel(name) + return layer, opts, name + @staticmethod def _store_json(value, file): file.write(bytes(json.dumps(value, indent=4, ensure_ascii=False), 'utf8')) diff --git a/src/vstarstack/library/stars/cut.py b/src/vstarstack/library/stars/cut.py index fe009c72..d7469c42 100644 --- a/src/vstarstack/library/stars/cut.py +++ b/src/vstarstack/library/stars/cut.py @@ -34,7 +34,7 @@ def cut_stars(image : vstarstack.library.data.DataFrame, mask = 1-mask layer = layer * mask - new_image.add_channel(layer, name, brightness=True) + new_image.add_channel(layer, name, **opts) if name in image.links["weight"]: weight,_ = image.get_channel(image.links["weight"][name]) weight = weight * mask From 0814894bed22f4042e253a37867d5439a4c60904 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sat, 31 Aug 2024 23:34:49 +0200 Subject: [PATCH 3/7] fix --- src/vstarstack/library/calibration/flat.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/vstarstack/library/calibration/flat.py b/src/vstarstack/library/calibration/flat.py index 4a45668e..aa9a1b0b 100644 --- a/src/vstarstack/library/calibration/flat.py +++ b/src/vstarstack/library/calibration/flat.py @@ -14,7 +14,6 @@ import cv2 import numpy as np -import skimage.filters import vstarstack.library.common import vstarstack.library.data @@ -23,6 +22,7 @@ import vstarstack.library.stars.cut import vstarstack.library.image_process.blur import vstarstack.library.merge.kappa_sigma +import vstarstack.library.image_process.normalize from vstarstack.library.image_process.blur import BlurredSource from vstarstack.library.image_process.nanmean_filter import nanmean_filter @@ -101,5 +101,8 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, continue layer = cv2.GaussianBlur(layer, (15, 15), 0) flat.add_channel(layer, channel, **opts) - + for channel in list(flat.get_channels()): + if not flat.get_channel_option(channel, "weight"): + continue + flat.remove_channel(channel) return flat From bc8eeff7422450628b5ee132fe969855a140a6c0 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sat, 31 Aug 2024 23:48:26 +0200 Subject: [PATCH 4/7] Optimize flat --- src/vstarstack/tool/calibration.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/vstarstack/tool/calibration.py b/src/vstarstack/tool/calibration.py index 5e4b8cc6..77af0d64 100644 --- a/src/vstarstack/tool/calibration.py +++ b/src/vstarstack/tool/calibration.py @@ -23,10 +23,10 @@ import vstarstack.library.calibration.flat def _process_file_flatten(input_fname : str, - flat_fname : str, + flat : vstarstack.library.data.DataFrame, output_fname : str): + print(f"Processing {input_fname}") dataframe = vstarstack.library.data.DataFrame.load(input_fname) - flat = vstarstack.library.data.DataFrame.load(flat_fname) result = vstarstack.library.calibration.flat.flatten(dataframe, flat) vstarstack.tool.common.check_dir_exists(output_fname) result.store(output_fname) @@ -41,13 +41,13 @@ def _process_file_remove_dark(input_fname : str, result.store(output_fname) def _process_dir_flatten(input_path : str, - flat_fname : str, + flat : vstarstack.library.data.DataFrame, output_path : str): files = vstarstack.tool.common.listfiles(input_path, ".zip") - for name, filename in files: - print(f"Processing {name}") - output_fname = os.path.join(output_path, name + ".zip") - _process_file_flatten(filename, flat_fname, output_fname) + files = vstarstack.tool.common.listfiles(input_path, ".zip") + with mp.Pool(vstarstack.tool.cfg.nthreads) as pool: + args = [(filename, flat, os.path.join(output_path, name + ".zip")) for name, filename in files] + pool.starmap(_process_file_flatten, args) def _process_dir_remove_dark(input_path : str, dark : vstarstack.library.data.DataFrame, @@ -62,10 +62,11 @@ def _process_flatten(_project : vstarstack.tool.cfg.Project, input_path = argv[0] flat_fname = argv[1] output_path = argv[2] + flat = vstarstack.library.data.DataFrame.load(flat_fname) if os.path.isdir(input_path): - _process_dir_flatten(input_path, flat_fname, output_path) + _process_dir_flatten(input_path, flat, output_path) else: - _process_file_flatten(input_path, flat_fname, output_path) + _process_file_flatten(input_path, flat, output_path) def _process_remove_dark(_project : vstarstack.tool.cfg.Project, argv : list[str]): From 3c0bd42bc435264a8b5789ebbe81285387db9cf1 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sun, 1 Sep 2024 00:04:11 +0200 Subject: [PATCH 5/7] fix --- src/vstarstack/library/calibration/flat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vstarstack/library/calibration/flat.py b/src/vstarstack/library/calibration/flat.py index aa9a1b0b..cb8a06e7 100644 --- a/src/vstarstack/library/calibration/flat.py +++ b/src/vstarstack/library/calibration/flat.py @@ -18,6 +18,7 @@ import vstarstack.library.common import vstarstack.library.data import vstarstack.library.merge +import vstarstack.library.merge.simple_add import vstarstack.library.stars.detect import vstarstack.library.stars.cut import vstarstack.library.image_process.blur @@ -50,7 +51,7 @@ def prepare_flat_simple(images : vstarstack.library.common.IImageSource, smooth_size += 1 source = BlurredSource(images, smooth_size) - dataframe = vstarstack.library.merge.simple_add(source) + dataframe = vstarstack.library.merge.simple_add.simple_add(source) return dataframe def calculate_median(image, weight, smooth_size): From 14d197d3fd1f88580818af7a5cbcb5334f6c5f32 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sun, 1 Sep 2024 00:06:57 +0200 Subject: [PATCH 6/7] fix --- src/vstarstack/library/merge/simple_mean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vstarstack/library/merge/simple_mean.py b/src/vstarstack/library/merge/simple_mean.py index 49175d88..ca61dd30 100644 --- a/src/vstarstack/library/merge/simple_mean.py +++ b/src/vstarstack/library/merge/simple_mean.py @@ -20,5 +20,5 @@ def mean(images : vstarstack.library.common.IImageSource) -> DataFrame: """Just mean of images""" - image = vstarstack.library.merge.simple_add(images) + image = vstarstack.library.merge.simple_add.simple_add(images) return vstarstack.library.image_process.normalize.normalize(image, deepcopy=False) From ef897e1327eff4e846384b3c9feb0c828cb6caa2 Mon Sep 17 00:00:00 2001 From: Vladislav Tsendrovskii Date: Sun, 1 Sep 2024 00:12:31 +0200 Subject: [PATCH 7/7] fix --- tests/test_merge.py | 5 +++-- tests/test_stars_pipeline.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_merge.py b/tests/test_merge.py index ba35f234..ada5cd27 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -19,6 +19,7 @@ import vstarstack.library.merge.kappa_sigma import vstarstack.library.merge.simple_add from vstarstack.library.common import ListImageSource +import vstarstack.library.merge.simple_mean dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -50,7 +51,7 @@ def test_simple_mean_1(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.simple_mean(source) + summ = vstarstack.library.merge.simple_mean.mean(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -80,7 +81,7 @@ def test_simple_mean_2(): noised.append(copy) source = ListImageSource(noised) - summ = vstarstack.library.merge.simple_mean(source) + summ = vstarstack.library.merge.simple_mean.mean(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] diff --git a/tests/test_stars_pipeline.py b/tests/test_stars_pipeline.py index a9eac063..1a4293d3 100644 --- a/tests/test_stars_pipeline.py +++ b/tests/test_stars_pipeline.py @@ -18,6 +18,7 @@ import vstarstack.library.common import vstarstack.library.loaders.classic +import vstarstack.library.merge.simple_add import vstarstack.library.stars.detect import vstarstack.library.stars.describe import vstarstack.library.stars.match @@ -123,7 +124,7 @@ def test_1(): assert len(moved) == 4 source = vstarstack.library.common.ListImageSource(moved) - merged = vstarstack.library.merge.simple_add(source) + merged = vstarstack.library.merge.simple_add.simple_add(source) layer,_ = merged.get_channel("L") merged_stars = vstarstack.library.stars.detect.detect_stars(layer) @@ -145,7 +146,7 @@ def test_1(): assert len(moved) == 4 source = vstarstack.library.common.ListImageSource(moved) - merged = vstarstack.library.merge.simple_add(source) + merged = vstarstack.library.merge.simple_add.simple_add(source) layer,_ = merged.get_channel("L") weight_name = merged.links["weight"]["L"]