Dev -> Stable 2.8.0

.github/workflows/benchmark.yml

@@ -51,7 +51,7 @@ jobs:
 
     # Upload benchmark results as artifacts
     - name: Upload benchmark results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: benchmark-results
         path: |

.github/workflows/distro_tests.yml

@@ -66,7 +66,7 @@ jobs:
           poetry run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
       - name: Upload Debug Logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: pytest-debug-logs-${{ env.OS_NAME }}
           path: pytest_debug.log

.github/workflows/tests.yml

@@ -39,7 +39,7 @@ jobs:
           poetry run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
           path: pytest_debug.log
@@ -58,6 +58,13 @@ jobs:
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -77,30 +84,97 @@ jobs:
           password: ${{ secrets.PYPI_API_TOKEN }}
       - name: Get BBOT version
         id: version
-        run: echo "BBOT_VERSION=$(poetry version | cut -d' ' -f2)" >> $GITHUB_OUTPUT
+        run: |
+          FULL_VERSION=$(poetry version | cut -d' ' -f2)
+          echo "BBOT_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT
+          # Extract major.minor (e.g., 2.7 from 2.7.1)
+          MAJOR_MINOR=$(echo "$FULL_VERSION" | cut -d'.' -f1-2)
+          echo "BBOT_VERSION_MAJOR_MINOR=$MAJOR_MINOR" >> $GITHUB_OUTPUT
+          # Extract major (e.g., 2 from 2.7.1)
+          MAJOR=$(echo "$FULL_VERSION" | cut -d'.' -f1)
+          echo "BBOT_VERSION_MAJOR=$MAJOR" >> $GITHUB_OUTPUT
       - name: Publish to Docker Hub (dev)
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-        uses: elgohr/Publish-Docker-Github-Action@v5
+        uses: docker/build-push-action@v6
         with:
-          name: blacklanternsecurity/bbot
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}"
+          push: true
+          context: .
+          tags: |
+            blacklanternsecurity/bbot:latest
+            blacklanternsecurity/bbot:dev
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}
       - name: Publish to Docker Hub (stable)
         if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
-        uses: elgohr/Publish-Docker-Github-Action@v5
+        uses: docker/build-push-action@v6
         with:
-          name: blacklanternsecurity/bbot
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}"
+          push: true
+          context: .
+          tags: |
+            blacklanternsecurity/bbot:stable
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}
+      - name: Publish Full Docker Image to Docker Hub (dev)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          file: Dockerfile.full
+          context: .
+          tags: |
+            blacklanternsecurity/bbot:latest-full
+            blacklanternsecurity/bbot:dev-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full
+      - name: Publish Full Docker Image to Docker Hub (stable)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          file: Dockerfile.full
+          context: .
+          tags: |
+            blacklanternsecurity/bbot:stable-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full
+            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full
       - name: Docker Hub Description
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
         uses: peter-evans/dockerhub-description@v5
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: blacklanternsecurity/bbot
+      - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest')
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        run: |
+          # Install jq for JSON processing
+          sudo apt-get update && sudo apt-get install -y jq
+
+          IMAGE="blacklanternsecurity/bbot"
+
+          # Clean up dev tags (keep 50 most recent)
+          for tag_pattern in "rc$" "rc-full$"; do
+            echo "Cleaning up tags ending with $tag_pattern..."
+
+            tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
+              "https://hub.docker.com/v2/repositories/$IMAGE/tags/?page_size=100")
+
+            tags_to_delete=$(echo "$tags_response" | jq -r --arg pattern "$tag_pattern" \
+              '.results[] | select(.name | test($pattern)) | [.last_updated, .name] | @tsv' | \
+              sort -r | tail -n +51 | cut -f2)
+
+            for tag in $tags_to_delete; do
+              echo "Deleting $IMAGE tag: $tag"
+              curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
+                "https://hub.docker.com/v2/repositories/$IMAGE/tags/$tag/"
+            done
+
+            echo "Cleanup completed for tags ending with $tag_pattern. Kept 50 most recent."
+          done
     outputs:
       BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}
   publish_docs:

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8

Dockerfile.full

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+ENV PIP_NO_CACHE_DIR=off
+
+WORKDIR /usr/src/bbot
+
+RUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo
+
+COPY . .
+
+RUN pip install .
+
+RUN bbot --install-all-deps
+
+WORKDIR /root
+
+ENTRYPOINT [ "bbot" ]

bbot/cli.py

@@ -7,7 +7,7 @@
 from bbot.errors import *
 from bbot import __version__
 from bbot.logger import log_to_stderr
-from bbot.core.helpers.misc import chain_lists
+from bbot.core.helpers.misc import chain_lists, rm_rf
 
 
 if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
 
         # --install-all-deps
         if options.install_all_deps:
-            all_modules = list(preset.module_loader.preloaded())
-            scan.helpers.depsinstaller.force_deps = True
-            succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            if failed:
-                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+            preloaded_modules = preset.module_loader.preloaded()
+            scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
+            output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
+            log.verbose("Creating dummy scan with all modules + output modules for deps installation")
+            dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
+            dummy_scan.helpers.depsinstaller.force_deps = True
+            log.info("Installing module dependencies")
+            await dummy_scan.load_modules()
+            log.verbose("Running module setups")
+            succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
+            # remove any leftovers from the dummy scan
+            rm_rf(dummy_scan.home, ignore_errors=True)
+            rm_rf(dummy_scan.temp_dir, ignore_errors=True)
+            if succeeded:
+                log.success(
+                    f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
+                )
+            if soft_failed or hard_failed:
+                failed = soft_failed + hard_failed
+                log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
                 return False
-            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
             return True
 
         scan_name = str(scan.name)

bbot/core/flags.py

@@ -6,6 +6,7 @@
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/core/helpers/web/web.py

@@ -267,7 +267,8 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg
         if not path:
             raise WordlistError(f"Invalid wordlist: {path}")
         if "cache_hrs" not in kwargs:
-            kwargs["cache_hrs"] = 720
+            # 4320 hrs = 180 days = 6 months
+            kwargs["cache_hrs"] = 4320
         if self.parent_helper.is_url(path):
             filename = await self.download(str(path), **kwargs)
             if filename is None:

bbot/core/modules.py

@@ -56,7 +56,6 @@ def __init__(self):
         self._shared_deps = dict(SHARED_DEPS)
 
         self.__preloaded = {}
-        self._modules = {}
         self._configs = {}
         self.flag_choices = set()
         self.all_module_choices = set()
@@ -463,7 +462,6 @@ def load_modules(self, module_names):
         for module_name in module_names:
             module = self.load_module(module_name)
             modules[module_name] = module
-            self._modules[module_name] = module
         return modules
 
     def load_module(self, module_name):

bbot/modules/apkpure.py

@@ -6,7 +6,7 @@
 class apkpure(BaseModule):
     watched_events = ["MOBILE_APP"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download android applications from apkpure.com",
         "created_date": "2024-10-11",

bbot/modules/base.py

@@ -213,6 +213,14 @@ async def setup(self):
 
         return True
 
+    async def setup_deps(self):
+        """
+        Similar to setup(), but reserved for installing dependencies not covered by Ansible.
+
+        This should always be used to install static dependencies like AI models, wordlists, etc.
+        """
+        return True
+
     async def handle_event(self, event, **kwargs):
         """Asynchronously handles incoming events that the module is configured to watch.
 
@@ -620,39 +628,26 @@ def start(self):
             name=f"{self.scan.name}.{self.name}._event_handler_watchdog()",
         )
 
-    async def _setup(self):
-        """
-        Asynchronously sets up the module by invoking its 'setup()' method.
-
-        This method catches exceptions during setup, sets the module's error state if necessary, and determines the
-        status code based on the result of the setup process.
-
-        Args:
-            None
-
-        Returns:
-            tuple: A tuple containing the module's name, status (True for success, False for hard-fail, None for soft-fail),
-            and an optional status message.
-
-        Raises:
-            Exception: Captured exceptions from the 'setup()' method are logged, but not propagated.
-
-        Notes:
-            - The 'setup()' method can return either a simple boolean status or a tuple of status and message.
-            - A WordlistError exception triggers a soft-fail status.
-            - The debug log will contain setup status information for the module.
-        """
+    async def _setup(self, deps_only=False):
+        """ """
         status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}
 
         status = False
         self.debug(f"Setting up module {self.name}")
         try:
-            result = await self.setup()
-            if type(result) == tuple and len(result) == 2:
-                status, msg = result
-            else:
-                status = result
-                msg = status_codes[status]
+            funcs = [self.setup_deps]
+            if not deps_only:
+                funcs.append(self.setup)
+            for func in funcs:
+                self.debug(f"Running {self.name}.{func.__name__}()")
+                result = await func()
+                if type(result) == tuple and len(result) == 2:
+                    status, msg = result
+                else:
+                    status = result
+                    msg = status_codes[status]
+                if status is False:
+                    break
             self.debug(f"Finished setting up module {self.name}")
         except Exception as e:
             self.set_error_state(f"Unexpected error during module setup: {e}", critical=True)

bbot/modules/dnsbrute.py

@@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum):
     dedup_strategy = "lowest_parent"
     _qsize = 10000
 
+    async def setup_deps(self):
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        # tell the dnsbrute helper to fetch the resolver file
+        await self.helpers.dns.brute.resolver_file()
+        return True
+
     async def setup(self):
         self.max_depth = max(1, self.config.get("max_depth", 5))
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
         self.wordlist_size = len(self.subdomain_list)
         return await super().setup()

bbot/modules/docker_pull.py

@@ -8,7 +8,7 @@
 class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",

bbot/modules/ffuf.py

@@ -37,12 +37,15 @@ class ffuf(BaseModule):
 
     in_scope_only = True
 
+    async def setup_deps(self):
+        self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
+        return True
+
     async def setup(self):
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist_url = self.config.get("wordlist", "")
         self.debug(f"Using wordlist [{wordlist_url}]")
-        self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
         self.rate = self.config.get("rate", 0)

bbot/modules/ffuf_shortnames.py

@@ -87,14 +87,17 @@ def find_common_prefixes(strings, minimum_set_length=4):
                     found_prefixes.add(prefix)
         return list(found_prefixes)
 
-    async def setup(self):
-        self.proxy = self.scan.web_config.get("http_proxy", "")
-        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
+    async def setup_deps(self):
         wordlist_extensions = self.config.get("wordlist_extensions", "")
         if not wordlist_extensions:
             wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
         self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
+        return True
+
+    async def setup(self):
+        self.proxy = self.scan.web_config.get("http_proxy", "")
+        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")