FourierFT: add alpha for dynamic n_frequency

Author: frutiemax92

src/peft/tuners/fourierft/config.py

@@ -185,6 +185,15 @@ class FourierFTConfig(PeftConfig):
         },
     )
 
+    alpha: float = field(
+        default=None,
+        metadata={
+            "help": (
+                "The alpha value dynamically sets the n_frequency = int(alpha * out_features * in_features)"
+            )
+        },
+    )
+
     def __post_init__(self):
         super().__post_init__()
         self.peft_type = PeftType.FOURIERFT

src/peft/tuners/fourierft/layer.py

@@ -29,7 +29,7 @@ class FourierFTLayer(BaseTunerLayer):
     # All names of other parameters that may contain adapter-related parameters
     other_param_names = ("fourierft_n_frequency", "fourierft_scaling", "fourierft_random_loc_seed")
 
-    def __init__(self, base_layer: nn.Module, **kwargs) -> None:
+    def __init__(self, base_layer: nn.Module, alpha, **kwargs) -> None:
         self.base_layer = base_layer
         self.fourierft_n_frequency = {}
         self.fourierft_scaling = {}
@@ -49,9 +49,14 @@ def __init__(self, base_layer: nn.Module, **kwargs) -> None:
                 base_layer.weight.ds_shape if hasattr(base_layer.weight, "ds_shape") else base_layer.weight.shape
             )
         elif isinstance(base_layer, nn.Conv2d):
-            pass
+            self.in_features = base_layer.in_channels
+            self.out_features = base_layer.out_channels
         else:
             raise ValueError(f"Unsupported layer type {type(base_layer)}")
+        
+        # apply alpha patch
+        if alpha:
+            kwargs['n_frequency'] = int(alpha * self.in_features * self.out_features)
 
     def update_layer(
         self, adapter_name, n_frequency, scaling, init_weights, random_loc_seed, inference_mode: bool = False, **kwargs
@@ -104,14 +109,16 @@ def __init__(
         base_layer,
         adapter_name: str,
         n_frequency: int = 1000,
+        alpha: float = None,
         scaling: float = 150.0,
         fan_in_fan_out: bool = False,  # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
         init_weights: Union[bool, str] = False,
         random_loc_seed: int = 777,
         **kwargs,
     ) -> None:
         super().__init__()
-        FourierFTLayer.__init__(self, base_layer, **kwargs)
+        FourierFTLayer.__init__(self, base_layer, alpha, **kwargs)
+
         self.fan_in_fan_out = fan_in_fan_out
         self._active_adapter = adapter_name
         self.update_layer(adapter_name, n_frequency, scaling, init_weights, random_loc_seed)
@@ -201,18 +208,17 @@ def __init__(
         base_layer,
         adapter_name: str,
         n_frequency: int = 1000,
+        alpha: float = None,
         scaling: float = 150.0,
         fan_in_fan_out: bool = False,  # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
         init_weights: Union[bool, str] = False,
         random_loc_seed: int = 777,
         **kwargs,
     ) -> None:
         super().__init__()
-        FourierFTLayer.__init__(self, base_layer, **kwargs)
+        FourierFTLayer.__init__(self, base_layer, alpha, **kwargs)
         self.fan_in_fan_out = fan_in_fan_out
         self._active_adapter = adapter_name
-        self.in_features = base_layer.in_channels
-        self.out_features = base_layer.out_channels
         self.kW = base_layer.kernel_size[0]
         self.kH = base_layer.kernel_size[1]
         self.stride = base_layer.stride

src/peft/tuners/fourierft/model.py

@@ -96,10 +96,12 @@ def _create_and_replace(
 
         n_frequency = fourierft_config.n_frequency_pattern.get(target_name_key, fourierft_config.n_frequency)
         scaling = fourierft_config.scaling
+        alpha = fourierft_config.alpha
         random_loc_seed = fourierft_config.random_loc_seed
         bias = hasattr(target, "bias") and target.bias is not None
         kwargs = {
             "n_frequency": n_frequency,
+            "alpha": alpha,
             "scaling": scaling,
             "fan_in_fan_out": fourierft_config.fan_in_fan_out,
             "init_weights": fourierft_config.init_weights,