Fix build flat from stars

src/vstarstack/library/calibration/dark.py

@@ -14,7 +14,7 @@
 
 import vstarstack.library.common
 import vstarstack.library.data
-import vstarstack.library.merge
+import vstarstack.library.merge.simple_mean
 
 def remove_dark(dataframe : vstarstack.library.data.DataFrame,
                 dark : vstarstack.library.data.DataFrame):
@@ -33,4 +33,4 @@ def remove_dark(dataframe : vstarstack.library.data.DataFrame,
 def prepare_darks(images : vstarstack.library.common.IImageSource
                   ) -> vstarstack.library.data.DataFrame:
     """Build dark frame"""
-    return vstarstack.library.merge.simple_mean(images)
+    return vstarstack.library.merge.simple_mean.mean(images)

src/vstarstack/library/calibration/flat.py

@@ -14,18 +14,20 @@
 
 import cv2
 import numpy as np
-import skimage.filters
 
 import vstarstack.library.common
 import vstarstack.library.data
 import vstarstack.library.merge
+import vstarstack.library.merge.simple_add
 import vstarstack.library.stars.detect
 import vstarstack.library.stars.cut
 import vstarstack.library.image_process.blur
+import vstarstack.library.merge.kappa_sigma
+import vstarstack.library.image_process.normalize
 
 from vstarstack.library.image_process.blur import BlurredSource
-from vstarstack.library.image_process.normalize import normalize
 from vstarstack.library.image_process.nanmean_filter import nanmean_filter
+from vstarstack.library.calibration.removehot import remove_hot_pixels
 
 def flatten(dataframe : vstarstack.library.data.DataFrame,
             flat : vstarstack.library.data.DataFrame):
@@ -49,7 +51,7 @@ def prepare_flat_simple(images : vstarstack.library.common.IImageSource,
         smooth_size += 1
 
     source = BlurredSource(images, smooth_size)
-    dataframe = vstarstack.library.merge.simple_add(source)
+    dataframe = vstarstack.library.merge.simple_add.simple_add(source)
     return dataframe
 
 def calculate_median(image, weight, smooth_size):
@@ -78,39 +80,30 @@ def prepare_flat_sky(images : vstarstack.library.common.IImageSource,
                      smooth_size : int
                      ) -> vstarstack.library.data.DataFrame:
     """Generate flat image"""
-    sum_layer = {}
-    sum_weight = {}
-    params = {}
+    no_star_images = []
     for dataframe in images.items():
         descs = []
-        params = dataframe.params
         for name in dataframe.get_channels():
             layer, opts = dataframe.get_channel(name)
             if not opts["brightness"]:
                 continue
             channel_descs = vstarstack.library.stars.detect.detect_stars(layer)
             descs += channel_descs
-        dataframe = vstarstack.library.image_process.blur.blur(dataframe, 5)
-        dataframe = normalize(dataframe)
-        nostars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs)
-        for name in nostars_dataframe.get_channels():
-            layer, opts = nostars_dataframe.get_channel(name)
-            if not opts["brightness"]:
-                continue
-            weight_name = nostars_dataframe.links["weight"][name]
-            weight, _ = nostars_dataframe.get_channel(weight_name)
-            if name not in sum_layer:
-                sum_layer[name] = layer
-                sum_weight[name] = weight
-            else:
-                sum_layer[name] += layer
-                sum_weight[name] += weight
 
-    flat = vstarstack.library.data.DataFrame(params=params)
-    for name, layer in sum_layer.items():
-        weight = sum_weight[name]
-        layer[np.where(weight == 0)] = 0
-        flat.add_channel(layer, name, brightness=True)
-        flat.add_channel(weight, "weight-"+name, weight=True)
-        flat.add_channel_link(name, "weight-"+name, "weight")
+        no_stars_dataframe = vstarstack.library.stars.cut.cut_stars(dataframe, descs)
+        no_stars_dataframe = remove_hot_pixels(no_stars_dataframe)
+        no_star_images.append(no_stars_dataframe)
+
+    no_star_source = vstarstack.library.common.ListImageSource(no_star_images)
+    flat = vstarstack.library.merge.kappa_sigma.kappa_sigma(no_star_source, 1, 1, 2)
+    for channel in flat.get_channels():
+        layer, opts = flat.get_channel(channel)
+        if not flat.get_channel_option(channel, "signal"):
+            continue
+        layer = cv2.GaussianBlur(layer, (15, 15), 0)
+        flat.add_channel(layer, channel, **opts)
+    for channel in list(flat.get_channels()):
+        if not flat.get_channel_option(channel, "weight"):
+            continue
+        flat.remove_channel(channel)
     return flat

src/vstarstack/library/calibration/removehot.py

@@ -0,0 +1,37 @@
+"""Remove hot pixels"""
+#
+# Copyright (c) 2024 Vladislav Tsendrovskii
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import numpy as np
+from vstarstack.library.data import DataFrame
+
+def remove_hot_pixels(image : DataFrame) -> DataFrame:
+    """
+    Remove hot pixels from image
+
+
+    """
+    for channel in image.get_channels():
+        layer, opts = image.get_channel(channel)
+        if not image.get_channel_option(channel, "signal"):
+            continue
+        hotpixels = np.where(layer >= np.median(layer)*3)
+        layer[hotpixels] = 0
+        image.add_channel(layer, channel, **opts)
+
+        weight, _, weight_channel = image.get_linked_channel(channel, "weight")
+        if weight_channel is not None:
+            weight[hotpixels] = 0
+            image.add_channel(weight, weight_channel, weight=True)
+    return image

src/vstarstack/library/data.py

@@ -178,6 +178,26 @@ def get_channel_option(self, channel : str, option : str) -> bool | None:
             return False
         return self.channels[channel]["options"][option]
 
+    def get_linked_channel(self, channel : str, link_type : str):
+        """
+        Get linked channel
+
+        Parameters:
+            channel (str) - channel name
+            link_type (str) - type of link
+
+        Returns:
+            None, None, None if no such link or no linked channel
+            layer, opts, name if such linked channel exists
+        """
+        if link_type not in self.links:
+            return None, None, None
+        if channel not in self.links[link_type]:
+            return None, None, None
+        name = self.links[link_type][channel]
+        layer, opts = self.get_channel(name)
+        return layer, opts, name
+
     @staticmethod
     def _store_json(value, file):
         file.write(bytes(json.dumps(value, indent=4, ensure_ascii=False), 'utf8'))

src/vstarstack/library/merge/__init__.py

@@ -1,17 +0,0 @@
-#
-# Copyright (c) 2023 Vladislav Tsendrovskii
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-
-from vstarstack.library.merge.kappa_sigma import kappa_sigma
-from vstarstack.library.merge.simple_add import simple_add
-from vstarstack.library.merge.simple_mean import mean as simple_mean

src/vstarstack/library/merge/simple_mean.py

@@ -20,5 +20,5 @@
 
 def mean(images : vstarstack.library.common.IImageSource) -> DataFrame:
     """Just mean of images"""
-    image = vstarstack.library.merge.simple_add(images)
+    image = vstarstack.library.merge.simple_add.simple_add(images)
     return vstarstack.library.image_process.normalize.normalize(image, deepcopy=False)

src/vstarstack/library/stars/cut.py

@@ -34,7 +34,7 @@ def cut_stars(image : vstarstack.library.data.DataFrame,
         mask = 1-mask
 
         layer = layer * mask
-        new_image.add_channel(layer, name, brightness=True)
+        new_image.add_channel(layer, name, **opts)
         if name in image.links["weight"]:
             weight,_ = image.get_channel(image.links["weight"][name])
             weight = weight * mask

src/vstarstack/tool/calibration.py

@@ -23,10 +23,10 @@
 import vstarstack.library.calibration.flat
 
 def _process_file_flatten(input_fname : str,
-                          flat_fname : str,
+                          flat : vstarstack.library.data.DataFrame,
                           output_fname : str):
+    print(f"Processing {input_fname}")
     dataframe = vstarstack.library.data.DataFrame.load(input_fname)
-    flat = vstarstack.library.data.DataFrame.load(flat_fname)
     result = vstarstack.library.calibration.flat.flatten(dataframe, flat)
     vstarstack.tool.common.check_dir_exists(output_fname)
     result.store(output_fname)
@@ -41,13 +41,13 @@ def _process_file_remove_dark(input_fname : str,
     result.store(output_fname)
 
 def _process_dir_flatten(input_path : str,
-                         flat_fname : str,
+                         flat : vstarstack.library.data.DataFrame,
                          output_path : str):
     files = vstarstack.tool.common.listfiles(input_path, ".zip")
-    for name, filename in files:
-        print(f"Processing {name}")
-        output_fname = os.path.join(output_path, name + ".zip")
-        _process_file_flatten(filename, flat_fname, output_fname)
+    files = vstarstack.tool.common.listfiles(input_path, ".zip")
+    with mp.Pool(vstarstack.tool.cfg.nthreads) as pool:
+        args = [(filename, flat, os.path.join(output_path, name + ".zip")) for name, filename in files]
+        pool.starmap(_process_file_flatten, args)
 
 def _process_dir_remove_dark(input_path : str,
                       dark : vstarstack.library.data.DataFrame,
@@ -62,10 +62,11 @@ def _process_flatten(_project : vstarstack.tool.cfg.Project,
     input_path = argv[0]
     flat_fname = argv[1]
     output_path = argv[2]
+    flat = vstarstack.library.data.DataFrame.load(flat_fname)
     if os.path.isdir(input_path):
-        _process_dir_flatten(input_path, flat_fname, output_path)
+        _process_dir_flatten(input_path, flat, output_path)
     else:
-        _process_file_flatten(input_path, flat_fname, output_path)
+        _process_file_flatten(input_path, flat, output_path)
 
 def _process_remove_dark(_project : vstarstack.tool.cfg.Project,
                          argv : list[str]):

src/vstarstack/tool/merge.py

@@ -16,6 +16,8 @@
 import vstarstack.library.data
 import vstarstack.library.merge
 
+import vstarstack.library.merge.simple_add
+import vstarstack.library.merge.kappa_sigma
 import vstarstack.tool.cfg
 import vstarstack.tool.usage
 import vstarstack.tool.common
@@ -33,7 +35,7 @@ def simple_add(project: vstarstack.tool.cfg.Project, argv: list):
 
     imgs = vstarstack.tool.common.listfiles(path_images, ".zip")
     filenames = [img[1] for img in imgs]
-    dataframe = vstarstack.library.merge.simple_add(FilesImageSource(filenames))
+    dataframe = vstarstack.library.merge.simple_add.simple_add(FilesImageSource(filenames))
     if dataframe is not None:
         vstarstack.tool.common.check_dir_exists(out)
         dataframe.store(out)
@@ -55,7 +57,7 @@ def sigma_clip(project: vstarstack.tool.cfg.Project, argv: list):
 
     imgs = vstarstack.tool.common.listfiles(path_images, ".zip")
     filenames = [img[1] for img in imgs]
-    dataframe = vstarstack.library.merge.kappa_sigma(FilesImageSource(filenames),
+    dataframe = vstarstack.library.merge.kappa_sigma.kappa_sigma(FilesImageSource(filenames),
                                                       kappa1,
                                                       kappa2,
                                                       sigma_steps)

tests/test_merge.py

@@ -16,8 +16,10 @@
 import os
 
 from vstarstack.library.loaders.classic import readjpeg
-import vstarstack.library.merge
+import vstarstack.library.merge.kappa_sigma
+import vstarstack.library.merge.simple_add
 from vstarstack.library.common import ListImageSource
+import vstarstack.library.merge.simple_mean
 
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
@@ -32,7 +34,7 @@ def test_simple_add():
     copy2 = original_image.copy()
     source = ListImageSource([copy1, copy2])
 
-    summ = vstarstack.library.merge.simple_add(source)
+    summ = vstarstack.library.merge.simple_add.simple_add(source)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -49,7 +51,7 @@ def test_simple_mean_1():
     copy2 = original_image.copy()
     source = ListImageSource([copy1, copy2])
 
-    summ = vstarstack.library.merge.simple_mean(source)
+    summ = vstarstack.library.merge.simple_mean.mean(source)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -79,7 +81,7 @@ def test_simple_mean_2():
         noised.append(copy)
     source = ListImageSource(noised)
 
-    summ = vstarstack.library.merge.simple_mean(source)
+    summ = vstarstack.library.merge.simple_mean.mean(source)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -95,7 +97,7 @@ def test_kappa_sigma_1():
     copy2 = original_image.copy()
     source = ListImageSource([copy1, copy2])
 
-    summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 0)
+    summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 0)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -112,7 +114,7 @@ def test_kappa_sigma_2():
     copy2 = original_image.copy()
     source = ListImageSource([copy1, copy2])
 
-    summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 1)
+    summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 1)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -127,7 +129,7 @@ def test_kappa_sigma_3():
     copy2 = original_image.copy()
     source = ListImageSource([copy1, copy2])
 
-    summ = vstarstack.library.merge.kappa_sigma(source, 1, 1, 2)
+    summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 1, 1, 2)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]
@@ -155,7 +157,7 @@ def test_kappa_sigma_4():
         noised.append(copy)
     source = ListImageSource(noised)
 
-    summ = vstarstack.library.merge.kappa_sigma(source, 3, 3, 5)
+    summ = vstarstack.library.merge.kappa_sigma.kappa_sigma(source, 3, 3, 5)
     summ_light, opts = summ.get_channel("L")
     wn = summ.links["weight"]["L"]
     summ_weight = summ.get_channel(wn)[0]

tests/test_stars_pipeline.py

@@ -18,6 +18,7 @@
 
 import vstarstack.library.common
 import vstarstack.library.loaders.classic
+import vstarstack.library.merge.simple_add
 import vstarstack.library.stars.detect
 import vstarstack.library.stars.describe
 import vstarstack.library.stars.match
@@ -123,7 +124,7 @@ def test_1():
     assert len(moved) == 4
 
     source = vstarstack.library.common.ListImageSource(moved)
-    merged = vstarstack.library.merge.simple_add(source)
+    merged = vstarstack.library.merge.simple_add.simple_add(source)
     layer,_ = merged.get_channel("L")
 
     merged_stars = vstarstack.library.stars.detect.detect_stars(layer)
@@ -145,7 +146,7 @@ def test_1():
     assert len(moved) == 4
 
     source = vstarstack.library.common.ListImageSource(moved)
-    merged = vstarstack.library.merge.simple_add(source)
+    merged = vstarstack.library.merge.simple_add.simple_add(source)
     layer,_ = merged.get_channel("L")
 
     weight_name = merged.links["weight"]["L"]