diff --git a/src/vstarstack/library/calibration/dark.py b/src/vstarstack/library/calibration/dark.py index beb09c23..c9c06d2d 100644 --- a/src/vstarstack/library/calibration/dark.py +++ b/src/vstarstack/library/calibration/dark.py @@ -14,7 +14,7 @@ import vstarstack.library.common import vstarstack.library.data -import vstarstack.library.merge +import vstarstack.library.merge.simple_mean def remove_dark(dataframe : vstarstack.library.data.DataFrame, dark : vstarstack.library.data.DataFrame): @@ -33,4 +33,4 @@ def remove_dark(dataframe : vstarstack.library.data.DataFrame, def prepare_darks(images : vstarstack.library.common.IImageSource ) -> vstarstack.library.data.DataFrame: """Build dark frame""" - return vstarstack.library.merge.simple_mean(images) + return vstarstack.library.merge.simple_mean.mean(images) diff --git a/src/vstarstack/library/calibration/flat.py b/src/vstarstack/library/calibration/flat.py index 9fd73fde..cb8a06e7 100644 --- a/src/vstarstack/library/calibration/flat.py +++ b/src/vstarstack/library/calibration/flat.py @@ -14,18 +14,20 @@ import cv2 import numpy as np -import skimage.filters import vstarstack.library.common import vstarstack.library.data import vstarstack.library.merge +import vstarstack.library.merge.simple_add import vstarstack.library.stars.detect import vstarstack.library.stars.cut import vstarstack.library.image_process.blur +import vstarstack.library.merge.kappa_sigma +import vstarstack.library.image_process.normalize from vstarstack.library.image_process.blur import BlurredSource -from vstarstack.library.image_process.normalize import normalize from vstarstack.library.image_process.nanmean_filter import nanmean_filter +from vstarstack.library.calibration.removehot import remove_hot_pixels def flatten(dataframe : vstarstack.library.data.DataFrame, flat : vstarstack.library.data.DataFrame): @@ -49,7 +51,7 @@ def prepare_flat_simple(images : vstarstack.library.common.IImageSource, smooth_size += 1 source = BlurredSource(images, smooth_size) - dataframe = vstarstack.library.merge.simple_add(source) + dataframe = vstarstack.library.merge.simple_add.simple_add(source) return dataframe def calculate_median(image, weight, smooth_size): @@ -78,39 +80,30 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource, smooth_size : int ) -> vstarstack.library.data.DataFrame: """Generate flat image""" - sum_layer = {} - sum_weight = {} - params = {} + no_star_images = [] for dataframe in images.items(): descs = [] - params = dataframe.params for name in dataframe.get_channels(): layer, opts = dataframe.get_channel(name) if not opts["brightness"]: continue channel_descs = vstarstack.library.stars.detect.detect_stars(layer) descs += channel_descs - dataframe = vstarstack.library.image_process.blur.blur(dataframe, 5) - dataframe = normalize(dataframe) - nostars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs) - for name in nostars_dataframe.get_channels(): - layer, opts = nostars_dataframe.get_channel(name) - if not opts["brightness"]: - continue - weight_name = nostars_dataframe.links["weight"][name] - weight, _ = nostars_dataframe.get_channel(weight_name) - if name not in sum_layer: - sum_layer[name] = layer - sum_weight[name] = weight - else: - sum_layer[name] += layer - sum_weight[name] += weight - flat = vstarstack.library.data.DataFrame(params=params) - for name, layer in sum_layer.items(): - weight = sum_weight[name] - layer[np.where(weight == 0)] = 0 - flat.add_channel(layer, name, brightness=True) - flat.add_channel(weight, "weight-"+name, weight=True) - flat.add_channel_link(name, "weight-"+name, "weight") + no_stars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs) + no_stars_dataframe = remove_hot_pixels(no_stars_dataframe) + no_star_images.append(no_stars_dataframe) + + no_star_source = vstarstack.library.common.ListImageSource(no_star_images) + flat = vstarstack.library.merge.kappa_sigma.kappa_sigma(no_star_source, 1, 1, 2) + for channel in flat.get_channels(): + layer, opts = flat.get_channel(channel) + if not flat.get_channel_option(channel, "signal"): + continue + layer = cv2.GaussianBlur(layer, (15, 15), 0) + flat.add_channel(layer, channel, **opts) + for channel in list(flat.get_channels()): + if not flat.get_channel_option(channel, "weight"): + continue + flat.remove_channel(channel) return flat diff --git a/src/vstarstack/library/calibration/removehot.py b/src/vstarstack/library/calibration/removehot.py new file mode 100644 index 00000000..7a139e5a --- /dev/null +++ b/src/vstarstack/library/calibration/removehot.py @@ -0,0 +1,37 @@ +"""Remove hot pixels""" +# +# Copyright (c) 2024 Vladislav Tsendrovskii +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import numpy as np +from vstarstack.library.data import DataFrame + +def remove_hot_pixels(image : DataFrame) -> DataFrame: + """ + Remove hot pixels from image + + + """ + for channel in image.get_channels(): + layer, opts = image.get_channel(channel) + if not image.get_channel_option(channel, "signal"): + continue + hotpixels = np.where(layer >= np.median(layer)*3) + layer[hotpixels] = 0 + image.add_channel(layer, channel, **opts) + + weight, _, weight_channel = image.get_linked_channel(channel, "weight") + if weight_channel is not None: + weight[hotpixels] = 0 + image.add_channel(weight, weight_channel, weight=True) + return image diff --git a/src/vstarstack/library/data.py b/src/vstarstack/library/data.py index 975f764d..e9aad2f0 100644 --- a/src/vstarstack/library/data.py +++ b/src/vstarstack/library/data.py @@ -178,6 +178,26 @@ def get_channel_option(self, channel : str, option : str) -> bool | None: return False return self.channels[channel]["options"][option] + def get_linked_channel(self, channel : str, link_type : str): + """ + Get linked channel + + Parameters: + channel (str) - channel name + link_type (str) - type of link + + Returns: + None, None, None if no such link or no linked channel + layer, opts, name if such linked channel exists + """ + if link_type not in self.links: + return None, None, None + if channel not in self.links[link_type]: + return None, None, None + name = self.links[link_type][channel] + layer, opts = self.get_channel(name) + return layer, opts, name + @staticmethod def _store_json(value, file): file.write(bytes(json.dumps(value, indent=4, ensure_ascii=False), 'utf8')) diff --git a/src/vstarstack/library/merge/__init__.py b/src/vstarstack/library/merge/__init__.py index 79d22d71..e69de29b 100644 --- a/src/vstarstack/library/merge/__init__.py +++ b/src/vstarstack/library/merge/__init__.py @@ -1,17 +0,0 @@ -# -# Copyright (c) 2023 Vladislav Tsendrovskii -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3 of the License. -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -from vstarstack.library.merge.kappa_sigma import kappa_sigma -from vstarstack.library.merge.simple_add import simple_add -from vstarstack.library.merge.simple_mean import mean as simple_mean diff --git a/src/vstarstack/library/merge/simple_mean.py b/src/vstarstack/library/merge/simple_mean.py index 49175d88..ca61dd30 100644 --- a/src/vstarstack/library/merge/simple_mean.py +++ b/src/vstarstack/library/merge/simple_mean.py @@ -20,5 +20,5 @@ def mean(images : vstarstack.library.common.IImageSource) -> DataFrame: """Just mean of images""" - image = vstarstack.library.merge.simple_add(images) + image = vstarstack.library.merge.simple_add.simple_add(images) return vstarstack.library.image_process.normalize.normalize(image, deepcopy=False) diff --git a/src/vstarstack/library/stars/cut.py b/src/vstarstack/library/stars/cut.py index fe009c72..d7469c42 100644 --- a/src/vstarstack/library/stars/cut.py +++ b/src/vstarstack/library/stars/cut.py @@ -34,7 +34,7 @@ def cut_stars(image : vstarstack.library.data.DataFrame, mask = 1-mask layer = layer * mask - new_image.add_channel(layer, name, brightness=True) + new_image.add_channel(layer, name, **opts) if name in image.links["weight"]: weight,_ = image.get_channel(image.links["weight"][name]) weight = weight * mask diff --git a/src/vstarstack/tool/calibration.py b/src/vstarstack/tool/calibration.py index 5e4b8cc6..77af0d64 100644 --- a/src/vstarstack/tool/calibration.py +++ b/src/vstarstack/tool/calibration.py @@ -23,10 +23,10 @@ import vstarstack.library.calibration.flat def _process_file_flatten(input_fname : str, - flat_fname : str, + flat : vstarstack.library.data.DataFrame, output_fname : str): + print(f"Processing {input_fname}") dataframe = vstarstack.library.data.DataFrame.load(input_fname) - flat = vstarstack.library.data.DataFrame.load(flat_fname) result = vstarstack.library.calibration.flat.flatten(dataframe, flat) vstarstack.tool.common.check_dir_exists(output_fname) result.store(output_fname) @@ -41,13 +41,13 @@ def _process_file_remove_dark(input_fname : str, result.store(output_fname) def _process_dir_flatten(input_path : str, - flat_fname : str, + flat : vstarstack.library.data.DataFrame, output_path : str): files = vstarstack.tool.common.listfiles(input_path, ".zip") - for name, filename in files: - print(f"Processing {name}") - output_fname = os.path.join(output_path, name + ".zip") - _process_file_flatten(filename, flat_fname, output_fname) + files = vstarstack.tool.common.listfiles(input_path, ".zip") + with mp.Pool(vstarstack.tool.cfg.nthreads) as pool: + args = [(filename, flat, os.path.join(output_path, name + ".zip")) for name, filename in files] + pool.starmap(_process_file_flatten, args) def _process_dir_remove_dark(input_path : str, dark : vstarstack.library.data.DataFrame, @@ -62,10 +62,11 @@ def _process_flatten(_project : vstarstack.tool.cfg.Project, input_path = argv[0] flat_fname = argv[1] output_path = argv[2] + flat = vstarstack.library.data.DataFrame.load(flat_fname) if os.path.isdir(input_path): - _process_dir_flatten(input_path, flat_fname, output_path) + _process_dir_flatten(input_path, flat, output_path) else: - _process_file_flatten(input_path, flat_fname, output_path) + _process_file_flatten(input_path, flat, output_path) def _process_remove_dark(_project : vstarstack.tool.cfg.Project, argv : list[str]): diff --git a/src/vstarstack/tool/merge.py b/src/vstarstack/tool/merge.py index ce3b14f5..f72fc337 100644 --- a/src/vstarstack/tool/merge.py +++ b/src/vstarstack/tool/merge.py @@ -16,6 +16,8 @@ import vstarstack.library.data import vstarstack.library.merge +import vstarstack.library.merge.simple_add +import vstarstack.library.merge.kappa_sigma import vstarstack.tool.cfg import vstarstack.tool.usage import vstarstack.tool.common @@ -33,7 +35,7 @@ def simple_add(project: vstarstack.tool.cfg.Project, argv: list): imgs = vstarstack.tool.common.listfiles(path_images, ".zip") filenames = [img[1] for img in imgs] - dataframe = vstarstack.library.merge.simple_add(FilesImageSource(filenames)) + dataframe = vstarstack.library.merge.simple_add.simple_add(FilesImageSource(filenames)) if dataframe is not None: vstarstack.tool.common.check_dir_exists(out) dataframe.store(out) @@ -55,7 +57,7 @@ def sigma_clip(project: vstarstack.tool.cfg.Project, argv: list): imgs = vstarstack.tool.common.listfiles(path_images, ".zip") filenames = [img[1] for img in imgs] - dataframe = vstarstack.library.merge.kappa_sigma(FilesImageSource(filenames), + dataframe = vstarstack.library.merge.kappa_sigma.kappa_sigma(FilesImageSource(filenames), kappa1, kappa2, sigma_steps) diff --git a/tests/test_merge.py b/tests/test_merge.py index 92918cf7..ada5cd27 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -16,8 +16,10 @@ import os from vstarstack.library.loaders.classic import readjpeg -import vstarstack.library.merge +import vstarstack.library.merge.kappa_sigma +import vstarstack.library.merge.simple_add from vstarstack.library.common import ListImageSource +import vstarstack.library.merge.simple_mean dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -32,7 +34,7 @@ def test_simple_add(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.simple_add(source) + summ = vstarstack.library.merge.simple_add.simple_add(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -49,7 +51,7 @@ def test_simple_mean_1(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.simple_mean(source) + summ = vstarstack.library.merge.simple_mean.mean(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -79,7 +81,7 @@ def test_simple_mean_2(): noised.append(copy) source = ListImageSource(noised) - summ = vstarstack.library.merge.simple_mean(source) + summ = vstarstack.library.merge.simple_mean.mean(source) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -95,7 +97,7 @@ def test_kappa_sigma_1(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 0) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 0) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -112,7 +114,7 @@ def test_kappa_sigma_2(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 1) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 1) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -127,7 +129,7 @@ def test_kappa_sigma_3(): copy2 = original_image.copy() source = ListImageSource([copy1, copy2]) - summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 2) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 2) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] @@ -155,7 +157,7 @@ def test_kappa_sigma_4(): noised.append(copy) source = ListImageSource(noised) - summ = vstarstack.library.merge.kappa_sigma(source, 3, 3, 5) + summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 3, 3, 5) summ_light, opts = summ.get_channel("L") wn = summ.links["weight"]["L"] summ_weight = summ.get_channel(wn)[0] diff --git a/tests/test_stars_pipeline.py b/tests/test_stars_pipeline.py index a9eac063..1a4293d3 100644 --- a/tests/test_stars_pipeline.py +++ b/tests/test_stars_pipeline.py @@ -18,6 +18,7 @@ import vstarstack.library.common import vstarstack.library.loaders.classic +import vstarstack.library.merge.simple_add import vstarstack.library.stars.detect import vstarstack.library.stars.describe import vstarstack.library.stars.match @@ -123,7 +124,7 @@ def test_1(): assert len(moved) == 4 source = vstarstack.library.common.ListImageSource(moved) - merged = vstarstack.library.merge.simple_add(source) + merged = vstarstack.library.merge.simple_add.simple_add(source) layer,_ = merged.get_channel("L") merged_stars = vstarstack.library.stars.detect.detect_stars(layer) @@ -145,7 +146,7 @@ def test_1(): assert len(moved) == 4 source = vstarstack.library.common.ListImageSource(moved) - merged = vstarstack.library.merge.simple_add(source) + merged = vstarstack.library.merge.simple_add.simple_add(source) layer,_ = merged.get_channel("L") weight_name = merged.links["weight"]["L"]