FIX Skip X-LoRA tests for transformers >= 4.49.0

Since the latest transformers release of v4.49.0, X-LoRA tests are
broken. The PR that caused it was:

huggingface/transformers#35724

For the time being, let's skip the X-LoRA tests if this transformers
version is detected and also advice users against using X-LoRA with this
transformers version.

Author: BenjaminBossan

docs/source/package_reference/xlora.md

@@ -16,6 +16,9 @@ rendered properly in your Markdown viewer.
 
 # X-LoRA
 
+> [!WARNING]
+> X-LoRA is broken for transformers version 4.49.0 and higher. It is recommended to use it with an older transformers version or to resort to other PEFT methods.
+
 Mixture of LoRA Experts ([X-LoRA](https://arxiv.org/abs/2402.07148)) is a PEFT method enabling sparse or dense mixture of LoRA experts based on a high granularity (token, layer, sequence) scalings matrix. This leverages frozen LoRA adapters and a frozen base model to drastically reduces the number of parameters that need to be fine-tuned.
 
 A unique aspect of X-LoRA is its versatility: it can be applied to any `transformers` base model with LoRA adapters. This means that, despite the mixture of experts strategy, no changes to the model code must be made.

src/peft/tuners/xlora/__init__.py

@@ -14,10 +14,10 @@
 
 from peft.utils import register_peft_method
 
-from .config import XLoraConfig
+from .config import XLORA_TRANSFORMERS_MAX_VERSION, XLoraConfig
 from .model import XLoraModel
 
 
-__all__ = ["XLoraConfig", "XLoraModel"]
+__all__ = ["XLORA_TRANSFORMERS_MAX_VERSION", "XLoraConfig", "XLoraModel"]
 
 register_peft_method(name="xlora", config_cls=XLoraConfig, model_cls=XLoraModel)

src/peft/tuners/xlora/config.py

@@ -13,14 +13,20 @@
 # limitations under the License.
 from __future__ import annotations
 
+import importlib
 import warnings
 from dataclasses import dataclass
 from typing import Optional
 
+from packaging import version
+
 from peft.config import PeftConfig
 from peft.utils.peft_types import PeftType
 
 
+XLORA_TRANSFORMERS_MAX_VERSION = "4.49.0"
+
+
 @dataclass
 class XLoraConfig(PeftConfig):
     r"""
@@ -79,6 +85,12 @@ def __post_init__(self):
         super().__post_init__()
         self.peft_type = PeftType.XLORA
 
+        if version.parse(importlib.metadata.version("transformers")) >= version.parse(XLORA_TRANSFORMERS_MAX_VERSION):
+            warnings.warn(
+                f"X-LoRA is currently broken with transformers {XLORA_TRANSFORMERS_MAX_VERSION}, it is recommended to "
+                "use an older transformers version or a different PEFT method"
+            )
+
         if self.hidden_size is None:
             warnings.warn(
                 "No value was provided for `hidden_size`. This will be set to 4096 by default, please ensure that this is correct."

tests/test_xlora.py

@@ -12,19 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import importlib
 import os
 
 import huggingface_hub
 import pytest
 import torch
+from packaging import version
 from safetensors.torch import load_file
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 from peft import LoraConfig, PeftType, TaskType, XLoraConfig, get_peft_model
 from peft.peft_model import PeftModel
+from peft.tuners.xlora import XLORA_TRANSFORMERS_MAX_VERSION
 from peft.utils import infer_device
 
 
+TRANSFORMERS_VERSION = version.parse(importlib.metadata.version("transformers"))
+
+
+@pytest.mark.skipif(
+    TRANSFORMERS_VERSION >= version.parse(XLORA_TRANSFORMERS_MAX_VERSION),
+    reason="X-LoRA is currently broken with the given transformers version, thus skipping tests",
+)
 class TestXlora:
     torch_device = infer_device()