feature: deferred loading and requirement pruning

docs/source/configurable.rst

@@ -200,8 +200,6 @@ probes and run each prompt just once:
 
 If we save this as ``latent1.yaml`` somewhere, then we can use it with ``garak --config latent1.yaml``.
 
-
-
 Using a Custom JSON Config
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -258,6 +256,26 @@ At plugin load, the plugin instance has attributes named in ``DEFAULT_PARAMS``
 automatically created, and populated with either values given in the supplied
 config, or the default.
 
+Fixed plugin parameters
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Some plugin parameters aren't intended to be altered at instantiation via config.
+These are the fixed plugin parameters, and are generally those not given in ``DEFAULT_PARAMS``.
+Descriptions of these are as follows (for a probe - other plugins are similar):
+
+* ``description`` - A short description of what the plugin does
+* ``active`` - Whether or not the plugin is active (i.e. selected) by default
+* ``doc_uri`` - Link to more information about the plugin
+* ``extended_detectors`` - Option detectors to use on probe results
+* ``extra_dependency_names`` - Extra Python modules that garka should import when instantiatng the plugin
+* ``goal`` - Brief description in imperative form of the probe's intent
+* ``modality`` - Which modalities the probe supports (as of Nov 2024 the list is ``text``, ``image``, ``audio``, ``video``, ``3d``)
+* ``parallelisable_attempts`` - Is the probe parallelisable? Recommended false if it has to use an LLM to develop attacks, particularly a local one
+* ``primary_detector`` - What detector should be used on the probe's outputs?
+* ``tags`` - List of tags applicable to the plugin, drawn from ``garak/data/tags.misp.tsv``
+* ``mod_time`` - Modification timestamp of the plugin source file used to generate this data
+
+
 .. _config_with_yaml:
 
 Configuring Plugins with YAML

docs/source/extending.rst

@@ -29,6 +29,9 @@ The recipe for writing a new plugin or plugin class isn't outlandish:
 * Start a new module inheriting from one of the base classes, e.g. :class:`garak.probes.base.Probe`
 * Override as little as possible.
 
+If you use custom modules not included in garak's default list, include these in the plugin's top-level ``extra_dependency_names`` parameter.
+Garak's plugin loader (``garak._plugins.load_plugin()``) will manage the import and inject the requested module as ``self.<module>``.
+
 
 Guides to writing plugins
 -------------------------

garak/_plugins.py

@@ -404,6 +404,28 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             ) from ve
         else:
             return False
+
+    full_plugin_name = ".".join((category, module_name, plugin_class_name))
+
+    # check cache for optional imports
+    if category in PLUGIN_TYPES:
+        extra_dependency_names = PluginCache.instance()[category][full_plugin_name][
+            "extra_dependency_names"
+        ]
+        if len(extra_dependency_names) > 0:
+            absent_modules = []
+            for dependency_module_name in extra_dependency_names:
+                for (
+                    dependency_path
+                ) in [  # support both plain names and also multi-point names e.g. langchain.llms
+                    ".".join(dependency_module_name.split(".")[: n + 1])
+                    for n in range(dependency_module_name.count(".") + 1)
+                ]:
+                    if importlib.util.find_spec(dependency_path) is None:
+                        absent_modules.append(dependency_module_name)
+            if len(absent_modules):
+                _import_failed(absent_modules, full_plugin_name)
+
     module_path = f"garak.{category}.{module_name}"
     try:
         mod = importlib.import_module(module_path)
@@ -434,6 +456,7 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
         if plugin_instance is None:
             plugin_instance = klass(config_root=config_root)
             PluginProvider.storeInstance(plugin_instance, config_root)
+
     except Exception as e:
         logging.warning(
             "Exception instantiating %s.%s: %s",
@@ -448,3 +471,44 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             return False
 
     return plugin_instance
+
+
+def load_optional_module(module_name: str):
+    try:
+        m = importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        requesting_module = Path(inspect.stack()[1].filename).name.replace(".py", "")
+        _import_failed([module_name], requesting_module)
+    return m
+
+
+def _import_failed(absent_modules: [str], calling_module: str):
+    quoted_module_list = "'" + "', '".join(absent_modules) + "'"
+    module_list = " ".join(absent_modules)
+    msg = f"⛔ Plugin '{calling_module}' requires Python modules which aren't installed/available: {quoted_module_list}"
+    hint = f"💡 Try 'pip install {module_list}' to get missing module."
+    logging.critical(msg)
+    print(msg + "\n" + hint)
+    raise ModuleNotFoundError(msg)
+
+
+def _load_deps(self, deps_override=list()):
+    # load external dependencies. should be invoked at construction and
+    # in _client_load (if used)
+    dep_names = deps_override if deps_override else self.extra_dependency_names
+    for extra_dependency in dep_names:
+        extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_")
+        if not hasattr(self, extra_dep_name) or getattr(self, extra_dep_name) is None:
+            setattr(
+                self,
+                extra_dep_name,
+                load_optional_module(extra_dependency),
+            )
+
+
+def _clear_deps(self):
+    # unload external dependencies from class. should be invoked before
+    # serialisation, esp. in _clear_client (if used)
+    for extra_dependency in self.extra_dependency_names:
+        extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_")
+        setattr(self, extra_dep_name, None)

garak/buffs/base.py

@@ -27,6 +27,8 @@ class Buff(Configurable):
     doc_uri = ""
     lang = None  # set of languages this buff should be constrained to
     active = True
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     DEFAULT_PARAMS = {}
 

garak/detectors/base.py

@@ -27,6 +27,8 @@ class Detector(Configurable):
     accuracy = None
     active = True
     tags = []  # list of taxonomy categories per the MISP format
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     # support mainstream any-to-any large models
     # legal element for str list `modality['in']`: 'text', 'image', 'audio', 'video', '3d'

garak/exception.py

@@ -14,7 +14,7 @@ class TargetNameMissingError(GarakException):
     """A generator requires target_name to be set, but it wasn't"""
 
 
-class GarakBackoffTrigger(GarakException):
+class GeneratorBackoffTrigger(GarakException):
     """Thrown when backoff should be triggered"""
 
 

garak/generators/azure.py

@@ -83,6 +83,7 @@ def _validate_env_var(self):
         return super()._validate_env_var()
 
     def _load_client(self):
+        self._load_deps()
         if self.target_name in openai_model_mapping:
             self.target_name = openai_model_mapping[self.target_name]
 

garak/generators/base.py

@@ -11,7 +11,7 @@
 from colorama import Fore, Style
 import tqdm
 
-from garak import _config
+from garak import _config, _plugins
 from garak.attempt import Message, Conversation
 from garak.configurable import Configurable
 from garak.exception import GarakException
@@ -46,6 +46,8 @@ class Generator(Configurable):
     supports_multiple_generations = (
         False  # can more than one generation be extracted per request?
     )
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     def __init__(self, name="", config_root=_config):
         self._load_config(config_root)
@@ -69,6 +71,10 @@ def __init__(self, name="", config_root=_config):
             f"🦜 loading {Style.BRIGHT}{Fore.LIGHTMAGENTA_EX}generator{Style.RESET_ALL}: {self.generator_family_name}: {self.name}"
         )
         logging.info("generator init: %s", self)
+        self._load_deps()
+
+    _load_deps = _plugins._load_deps
+    _clear_deps = _plugins._clear_deps
 
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1

garak/generators/bedrock.py

@@ -42,6 +42,7 @@ class BedrockGenerator(Generator):
     active = True
     generator_family_name = "Bedrock"
     supports_multiple_generations = False
+    extra_dependency_names = ["boto3", "botocore"]
 
     DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | {
         "temperature": 0.7,
@@ -106,7 +107,7 @@ def __init__(self, name="", config_root=_config):
 
     def _validate_env_var(self):
         """Validate and set region from environment variables if not configured.
-        
+
         Checks AWS_REGION and AWS_DEFAULT_REGION environment variables only if
         the region parameter is still at its default value.
         """
@@ -115,23 +116,16 @@ def _validate_env_var(self):
             if env_region:
                 logging.info(f"Using AWS region from environment: {env_region}")
                 self.region = env_region
-        
+
         return super()._validate_env_var()
 
     def _load_client(self):
         """Load and configure the boto3 bedrock-runtime client.
 
         Uses boto3's standard credential chain for authentication.
         """
-        try:
-            import boto3
-        except ImportError:
-            raise ImportError(
-                "boto3 is required for the Bedrock generator. "
-                "Install it with: pip install boto3"
-            )
-
-        self.client = boto3.client(
+
+        self.client = self.boto3.client(
             service_name="bedrock-runtime",
             region_name=self.region,
         )
@@ -142,6 +136,8 @@ def _clear_client(self):
         """Clear the boto3 client to enable object pickling."""
         if hasattr(self, "client"):
             self.client = None
+        for module_name in self.extra_dependency_names:
+            setattr(self, module_name, None)
 
     def __getstate__(self):
         """Prepare object for pickling by clearing the boto3 client."""
@@ -156,13 +152,13 @@ def __setstate__(self, state):
     @staticmethod
     def _conversation_to_list(conversation: Conversation) -> list[dict]:
         """Convert Conversation object to Bedrock Converse API message format.
-        
+
         AWS Bedrock expects messages in the format:
         {"role": "user", "content": [{"text": "message text"}]}
-        
+
         Args:
             conversation: Conversation object to convert
-            
+
         Returns:
             List of message dictionaries in Bedrock format
         """
@@ -174,7 +170,7 @@ def _conversation_to_list(conversation: Conversation) -> list[dict]:
 
     @backoff.on_exception(
         backoff.fibo,
-        garak.exception.GarakBackoffTrigger,
+        garak.exception.GeneratorBackoffTrigger,
         max_value=70,
     )
     def _call_model(
@@ -246,20 +242,20 @@ def _call_model(
             return [Message(text=text)]
 
         except Exception as e:
-            from botocore.exceptions import ClientError
 
-            if isinstance(e, ClientError):
+            if isinstance(e, self.botocore.exceptions.ClientError):
                 error_code = e.response.get("Error", {}).get("Code", "")
                 error_message = e.response.get("Error", {}).get("Message", "")
 
                 logging.error(f"Bedrock API error [{error_code}]: {error_message}")
 
                 if error_code in ["ThrottlingException", "ServiceUnavailableException"]:
-                    raise garak.exception.GarakBackoffTrigger from e
+                    raise garak.exception.GeneratorBackoffTrigger from e
 
                 return [None]
 
             logging.exception("Error calling Bedrock model")
             return [None]
 
-DEFAULT_CLASS = "BedrockGenerator"
+
+DEFAULT_CLASS = "BedrockGenerator"

garak/generators/cohere.py

@@ -16,12 +16,11 @@
 from typing import List, Union
 
 import backoff
-from cohere.core.api_error import ApiError
-import cohere
 import tqdm
 
 from garak import _config
 from garak.attempt import Message, Conversation
+from garak.exception import GeneratorBackoffTrigger
 from garak.generators.base import Generator
 
 
@@ -52,6 +51,8 @@ class CohereGenerator(Generator):
         "api_version": "v2",  # "v1" for legacy generate API, "v2" for chat API (recommended)
     }
 
+    extra_dependency_names = ["cohere"]
+
     generator_family_name = "Cohere"
 
     def __init__(self, name="command", config_root=_config):
@@ -74,11 +75,11 @@ def __init__(self, name="command", config_root=_config):
         # Initialize appropriate client based on API version
         # Following Cohere's guidance to use Client() for v1 and ClientV2() for v2
         if self.api_version == "v1":
-            self.generator = cohere.Client(api_key=self.api_key)
+            self.generator = self.cohere.Client(api_key=self.api_key)
         else:  # api_version == "v2"
-            self.generator = cohere.ClientV2(api_key=self.api_key)
+            self.generator = self.cohere.ClientV2(api_key=self.api_key)
 
-    @backoff.on_exception(backoff.fibo, ApiError, max_value=70)
+    @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70)
     def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
         """Empty prompts raise API errors (e.g. invalid request: prompt must be at least 1 token long).
         We catch these using the ApiError base class in Cohere v5+.
@@ -127,9 +128,17 @@ def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
                                 "Chat response structure doesn't match expected format"
                             )
                             responses.append(str(response))
-                    except ApiError as e:
-                        raise e  # bubble up ApiError for backoff handling
                     except Exception as e:
+
+                        backoff_exception_types = (
+                            self.cohere.errors.GatewayTimeoutError,
+                            self.cohere.errors.TooManyRequestsError,
+                            self.cohere.errors.ServiceUnavailableError,
+                            self.cohere.errors.InternalServerError,
+                        )
+                        for backoff_exception in backoff_exception_types:
+                            if isinstance(e, backoff_exception):
+                                raise GeneratorBackoffTrigger from e  # bubble up ApiError for backoff handling
                         logging.error(f"Chat API error: {e}")
                         responses.append(None)
 

garak/generators/groq.py

@@ -40,6 +40,7 @@ class GroqChat(OpenAICompatible):
     generator_family_name = "Groq"
 
     def _load_client(self):
+        self._load_deps()
         self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key)
         if self.name in ("", None):
             raise ValueError(

garak/generators/guardrails.py

@@ -17,27 +17,22 @@ class NeMoGuardrails(Generator):
 
     supports_multiple_generations = False
     generator_family_name = "Guardrails"
+    extra_dependency_names = ["nemoguardrails"]
 
     def __init__(self, name="", config_root=_config):
-        # another class that may need to skip testing due to non required dependency
-        try:
-            from nemoguardrails import RailsConfig, LLMRails
-        except ImportError as e:
-            raise NameError(
-                "You must first install NeMo Guardrails using `pip install nemoguardrails`."
-            ) from e
 
         self.name = name
         self._load_config(config_root)
         self.fullname = f"Guardrails {self.name}"
 
-        # Currently, we use the target_name as the path to the config
-        with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
-            config = RailsConfig.from_path(self.name)
-            self.rails = LLMRails(config=config)
-
         super().__init__(self.name, config_root=config_root)
 
+        set_verbose = self.nemoguardrails.logging.verbose.set_verbose
+        # Currently, we use the model_name as the path to the config
+        with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
+            config = self.nemoguardrails.RailsConfig.from_path(self.name)
+            self.rails = self.nemoguardrails.LLMRails(config=config)
+
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1
     ) -> List[Union[Message, None]]: