Better --install-all-deps

.github/workflows/tests.yml

@@ -58,6 +58,11 @@ jobs:
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -83,24 +88,59 @@ jobs:
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
           name: blacklanternsecurity/bbot
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
           tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}"
       - name: Publish to Docker Hub (stable)
         if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
           name: blacklanternsecurity/bbot
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
           tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}"
+      - name: Publish Full Docker Image to Docker Hub (dev)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: blacklanternsecurity/bbot
+          dockerfile: Dockerfile.full
+          tags: "latest-full,dev-full,${{ steps.version.outputs.BBOT_VERSION }}-full"
+      - name: Publish Full Docker Image to Docker Hub (stable)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: blacklanternsecurity/bbot
+          dockerfile: Dockerfile.full
+          tags: "stable-full,${{ steps.version.outputs.BBOT_VERSION }}-full"
       - name: Docker Hub Description
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-        uses: peter-evans/dockerhub-description@v4
+        uses: peter-evans/dockerhub-description@v5
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
           repository: blacklanternsecurity/bbot
+      - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest')
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        run: |
+          # Install jq for JSON processing
+          sudo apt-get update && sudo apt-get install -y jq
+
+          IMAGE="blacklanternsecurity/bbot"
+
+          # Clean up dev tags (keep 50 most recent)
+          for tag_pattern in "rc$" "rc-full$"; do
+            echo "Cleaning up tags ending with $tag_pattern..."
+
+            tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
+              "https://hub.docker.com/v2/repositories/$IMAGE/tags/?page_size=100")
+
+            tags_to_delete=$(echo "$tags_response" | jq -r --arg pattern "$tag_pattern" \
+              '.results[] | select(.name | test($pattern)) | [.last_updated, .name] | @tsv' | \
+              sort -r | tail -n +51 | cut -f2)
+
+            for tag in $tags_to_delete; do
+              echo "Deleting $IMAGE tag: $tag"
+              curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
+                "https://hub.docker.com/v2/repositories/$IMAGE/tags/$tag/"
+            done
+
+            echo "Cleanup completed for tags ending with $tag_pattern. Kept 50 most recent."
+          done
     outputs:
       BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}
   publish_docs:

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8

Dockerfile.full

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+ENV PIP_NO_CACHE_DIR=off
+
+WORKDIR /usr/src/bbot
+
+RUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo
+
+COPY . .
+
+RUN pip install .
+
+RUN bbot --install-all-deps
+
+WORKDIR /root
+
+ENTRYPOINT [ "bbot" ]

bbot/cli.py

@@ -7,7 +7,7 @@
 from bbot.errors import *
 from bbot import __version__
 from bbot.logger import log_to_stderr
-from bbot.core.helpers.misc import chain_lists
+from bbot.core.helpers.misc import chain_lists, rm_rf
 
 
 if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
 
         # --install-all-deps
         if options.install_all_deps:
-            all_modules = list(preset.module_loader.preloaded())
-            scan.helpers.depsinstaller.force_deps = True
-            succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            if failed:
-                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+            preloaded_modules = preset.module_loader.preloaded()
+            scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
+            output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
+            log.verbose("Creating dummy scan with all modules + output modules for deps installation")
+            dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
+            dummy_scan.helpers.depsinstaller.force_deps = True
+            log.info("Installing module dependencies")
+            await dummy_scan.load_modules()
+            log.verbose("Running module setups")
+            succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
+            # remove any leftovers from the dummy scan
+            rm_rf(dummy_scan.home, ignore_errors=True)
+            rm_rf(dummy_scan.temp_dir, ignore_errors=True)
+            if succeeded:
+                log.success(
+                    f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
+                )
+            if soft_failed or hard_failed:
+                failed = soft_failed + hard_failed
+                log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
                 return False
-            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
             return True
 
         scan_name = str(scan.name)

bbot/core/helpers/web/web.py

@@ -267,7 +267,8 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg
         if not path:
             raise WordlistError(f"Invalid wordlist: {path}")
         if "cache_hrs" not in kwargs:
-            kwargs["cache_hrs"] = 720
+            # 4320 hrs = 180 days = 6 months
+            kwargs["cache_hrs"] = 4320
         if self.parent_helper.is_url(path):
             filename = await self.download(str(path), **kwargs)
             if filename is None:

bbot/core/modules.py

@@ -56,7 +56,6 @@ def __init__(self):
         self._shared_deps = dict(SHARED_DEPS)
 
         self.__preloaded = {}
-        self._modules = {}
         self._configs = {}
         self.flag_choices = set()
         self.all_module_choices = set()
@@ -463,7 +462,6 @@ def load_modules(self, module_names):
         for module_name in module_names:
             module = self.load_module(module_name)
             modules[module_name] = module
-            self._modules[module_name] = module
         return modules
 
     def load_module(self, module_name):

bbot/modules/base.py

@@ -213,6 +213,14 @@ async def setup(self):
 
         return True
 
+    async def setup_deps(self):
+        """
+        Similar to setup(), but reserved for installing dependencies not covered by Ansible.
+
+        This should always be used to install static dependencies like AI models, wordlists, etc.
+        """
+        return True
+
     async def handle_event(self, event, **kwargs):
         """Asynchronously handles incoming events that the module is configured to watch.
 
@@ -620,39 +628,26 @@ def start(self):
             name=f"{self.scan.name}.{self.name}._event_handler_watchdog()",
         )
 
-    async def _setup(self):
-        """
-        Asynchronously sets up the module by invoking its 'setup()' method.
-
-        This method catches exceptions during setup, sets the module's error state if necessary, and determines the
-        status code based on the result of the setup process.
-
-        Args:
-            None
-
-        Returns:
-            tuple: A tuple containing the module's name, status (True for success, False for hard-fail, None for soft-fail),
-            and an optional status message.
-
-        Raises:
-            Exception: Captured exceptions from the 'setup()' method are logged, but not propagated.
-
-        Notes:
-            - The 'setup()' method can return either a simple boolean status or a tuple of status and message.
-            - A WordlistError exception triggers a soft-fail status.
-            - The debug log will contain setup status information for the module.
-        """
+    async def _setup(self, deps_only=False):
+        """ """
         status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}
 
         status = False
         self.debug(f"Setting up module {self.name}")
         try:
-            result = await self.setup()
-            if type(result) == tuple and len(result) == 2:
-                status, msg = result
-            else:
-                status = result
-                msg = status_codes[status]
+            funcs = [self.setup_deps]
+            if not deps_only:
+                funcs.append(self.setup)
+            for func in funcs:
+                self.debug(f"Running {self.name}.{func.__name__}()")
+                result = await func()
+                if type(result) == tuple and len(result) == 2:
+                    status, msg = result
+                else:
+                    status = result
+                    msg = status_codes[status]
+                if status is False:
+                    break
             self.debug(f"Finished setting up module {self.name}")
         except Exception as e:
             self.set_error_state(f"Unexpected error during module setup: {e}", critical=True)

bbot/modules/dnsbrute.py

@@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum):
     dedup_strategy = "lowest_parent"
     _qsize = 10000
 
+    async def setup_deps(self):
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        # tell the dnsbrute helper to fetch the resolver file
+        await self.helpers.dns.brute.resolver_file()
+        return True
+
     async def setup(self):
         self.max_depth = max(1, self.config.get("max_depth", 5))
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
         self.wordlist_size = len(self.subdomain_list)
         return await super().setup()

bbot/modules/ffuf.py

@@ -37,12 +37,15 @@ class ffuf(BaseModule):
 
     in_scope_only = True
 
+    async def setup_deps(self):
+        self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
+        return True
+
     async def setup(self):
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist_url = self.config.get("wordlist", "")
         self.debug(f"Using wordlist [{wordlist_url}]")
-        self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
         self.rate = self.config.get("rate", 0)

bbot/modules/ffuf_shortnames.py

@@ -87,14 +87,17 @@ def find_common_prefixes(strings, minimum_set_length=4):
                     found_prefixes.add(prefix)
         return list(found_prefixes)
 
-    async def setup(self):
-        self.proxy = self.scan.web_config.get("http_proxy", "")
-        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
+    async def setup_deps(self):
         wordlist_extensions = self.config.get("wordlist_extensions", "")
         if not wordlist_extensions:
             wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
         self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
+        return True
+
+    async def setup(self):
+        self.proxy = self.scan.web_config.get("http_proxy", "")
+        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")

bbot/modules/filedownload.py

@@ -94,6 +94,12 @@ class filedownload(BaseModule):
 
     scope_distance_modifier = 3
 
+    async def setup_deps(self):
+        self.mime_db_file = await self.helpers.wordlist(
+            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
+        )
+        return True
+
     async def setup(self):
         self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
         self.max_filesize = self.config.get("max_filesize", "10MB")
@@ -105,9 +111,6 @@ async def setup(self):
         else:
             self.download_dir = self.scan.temp_dir / "filedownload"
         self.helpers.mkdir(self.download_dir)
-        self.mime_db_file = await self.helpers.wordlist(
-            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
-        )
         self.mime_db = {}
         with open(self.mime_db_file) as f:
             mime_db = json.load(f)

bbot/modules/medusa.py

@@ -1,6 +1,5 @@
 import re
 from bbot.modules.base import BaseModule
-from bbot.errors import WordlistError
 
 
 class medusa(BaseModule):
@@ -102,13 +101,11 @@ class medusa(BaseModule):
         },
     ]
 
-    async def setup(self):
-        # Try to cache wordlist
-        try:
-            self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
-        except WordlistError as e:
-            return False, f"Error retrieving wordlist: {e}"
+    async def setup_deps(self):
+        self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
+        return True
 
+    async def setup(self):
         self.password_match_regex = re.compile(r"Password:\s*(\S+)")
         self.success_indicator_match_regex = re.compile(r"\[([^\]]+)\]\s*$")
 

bbot/modules/paramminer_headers.py

@@ -82,18 +82,21 @@ class paramminer_headers(BaseModule):
 
     header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$")
 
-    async def setup(self):
-        self.recycle_words = self.config.get("recycle_words", True)
-        self.event_dict = {}
-        self.already_checked = set()
+    async def setup_deps(self):
         wordlist = self.config.get("wordlist", "")
         if not wordlist:
             wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}"
+        self.wordlist_file = await self.helpers.wordlist(wordlist)
         self.debug(f"Using wordlist: [{wordlist}]")
+        return True
+
+    async def setup(self):
+        self.recycle_words = self.config.get("recycle_words", True)
+        self.event_dict = {}
+        self.already_checked = set()
+
         self.wl = {
-            h.strip().lower()
-            for h in self.helpers.read_file(await self.helpers.wordlist(wordlist))
-            if len(h) > 0 and "%" not in h
+            h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and "%" not in h
         }
 
         # check against the boring list (if the option is set)

bbot/modules/trufflehog.py

@@ -41,11 +41,14 @@ class trufflehog(BaseModule):
 
     scope_distance_modifier = 2
 
-    async def setup(self):
-        self.verified = self.config.get("only_verified", True)
+    async def setup_deps(self):
         self.config_file = self.config.get("config", "")
         if self.config_file:
             self.config_file = await self.helpers.wordlist(self.config_file)
+        return True
+
+    async def setup(self):
+        self.verified = self.config.get("only_verified", True)
         self.concurrency = int(self.config.get("concurrency", 8))
 
         self.deleted_forks = self.config.get("deleted_forks", False)