Skip to content

Commit 08147dc

Browse files
TheTechromancerTheTechromancer
authored
Merge pull request #2730 from blacklanternsecurity/better-install-deps
Better --install-all-deps
2 parents f9f6450 + 18cc96a commit 08147dc

File tree

18 files changed

+198
-81
lines changed

18 files changed

+198
-81
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ jobs:
5858
- uses: actions/checkout@v5
5959
with:
6060
fetch-depth: 0
61+
- name: Login to Docker Hub
62+
uses: docker/login-action@v3
63+
with:
64+
username: ${{ secrets.DOCKER_USERNAME }}
65+
password: ${{ secrets.DOCKER_PASSWORD }}
6166
- name: Set up Python
6267
uses: actions/setup-python@v6
6368
with:
@@ -83,24 +88,59 @@ jobs:
8388
uses: elgohr/Publish-Docker-Github-Action@v5
8489
with:
8590
name: blacklanternsecurity/bbot
86-
username: ${{ secrets.DOCKER_USERNAME }}
87-
password: ${{ secrets.DOCKER_PASSWORD }}
8891
tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}"
8992
- name: Publish to Docker Hub (stable)
9093
if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
9194
uses: elgohr/Publish-Docker-Github-Action@v5
9295
with:
9396
name: blacklanternsecurity/bbot
94-
username: ${{ secrets.DOCKER_USERNAME }}
95-
password: ${{ secrets.DOCKER_PASSWORD }}
9697
tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}"
98+
- name: Publish Full Docker Image to Docker Hub (dev)
99+
if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
100+
uses: elgohr/Publish-Docker-Github-Action@v5
101+
with:
102+
name: blacklanternsecurity/bbot
103+
dockerfile: Dockerfile.full
104+
tags: "latest-full,dev-full,${{ steps.version.outputs.BBOT_VERSION }}-full"
105+
- name: Publish Full Docker Image to Docker Hub (stable)
106+
if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
107+
uses: elgohr/Publish-Docker-Github-Action@v5
108+
with:
109+
name: blacklanternsecurity/bbot
110+
dockerfile: Dockerfile.full
111+
tags: "stable-full,${{ steps.version.outputs.BBOT_VERSION }}-full"
97112
- name: Docker Hub Description
98113
if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
99114
uses: peter-evans/dockerhub-description@v5
100115
with:
101-
username: ${{ secrets.DOCKER_USERNAME }}
102-
password: ${{ secrets.DOCKER_PASSWORD }}
103116
repository: blacklanternsecurity/bbot
117+
- name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest')
118+
if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
119+
run: |
120+
# Install jq for JSON processing
121+
sudo apt-get update && sudo apt-get install -y jq
122+
123+
IMAGE="blacklanternsecurity/bbot"
124+
125+
# Clean up dev tags (keep 50 most recent)
126+
for tag_pattern in "rc$" "rc-full$"; do
127+
echo "Cleaning up tags ending with $tag_pattern..."
128+
129+
tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
130+
"https://hub.docker.com/v2/repositories/$IMAGE/tags/?page_size=100")
131+
132+
tags_to_delete=$(echo "$tags_response" | jq -r --arg pattern "$tag_pattern" \
133+
'.results[] | select(.name | test($pattern)) | [.last_updated, .name] | @tsv' | \
134+
sort -r | tail -n +51 | cut -f2)
135+
136+
for tag in $tags_to_delete; do
137+
echo "Deleting $IMAGE tag: $tag"
138+
curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \
139+
"https://hub.docker.com/v2/repositories/$IMAGE/tags/$tag/"
140+
done
141+
142+
echo "Cleanup completed for tags ending with $tag_pattern. Kept 50 most recent."
143+
done
104144
outputs:
105145
BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}
106146
publish_docs:

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.10-slim
1+
FROM python:3.11-slim
22

33
ENV LANG=C.UTF-8
44
ENV LC_ALL=C.UTF-8

‎Dockerfile.full‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM python:3.11-slim
2+
3+
ENV LANG=C.UTF-8
4+
ENV LC_ALL=C.UTF-8
5+
ENV PIP_NO_CACHE_DIR=off
6+
7+
WORKDIR /usr/src/bbot
8+
9+
RUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo
10+
11+
COPY . .
12+
13+
RUN pip install .
14+
15+
RUN bbot --install-all-deps
16+
17+
WORKDIR /root
18+
19+
ENTRYPOINT [ "bbot" ]

‎bbot/cli.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from bbot.errors import *
88
from bbot import __version__
99
from bbot.logger import log_to_stderr
10-
from bbot.core.helpers.misc import chain_lists
10+
from bbot.core.helpers.misc import chain_lists, rm_rf
1111

1212

1313
if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
173173

174174
# --install-all-deps
175175
if options.install_all_deps:
176-
all_modules = list(preset.module_loader.preloaded())
177-
scan.helpers.depsinstaller.force_deps = True
178-
succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
179-
if failed:
180-
log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
176+
preloaded_modules = preset.module_loader.preloaded()
177+
scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
178+
output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
179+
log.verbose("Creating dummy scan with all modules + output modules for deps installation")
180+
dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
181+
dummy_scan.helpers.depsinstaller.force_deps = True
182+
log.info("Installing module dependencies")
183+
await dummy_scan.load_modules()
184+
log.verbose("Running module setups")
185+
succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
186+
# remove any leftovers from the dummy scan
187+
rm_rf(dummy_scan.home, ignore_errors=True)
188+
rm_rf(dummy_scan.temp_dir, ignore_errors=True)
189+
if succeeded:
190+
log.success(
191+
f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
192+
)
193+
if soft_failed or hard_failed:
194+
failed = soft_failed + hard_failed
195+
log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
181196
return False
182-
log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
183197
return True
184198

185199
scan_name = str(scan.name)

‎bbot/core/helpers/web/web.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,8 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg
267267
if not path:
268268
raise WordlistError(f"Invalid wordlist: {path}")
269269
if "cache_hrs" not in kwargs:
270-
kwargs["cache_hrs"] = 720
270+
# 4320 hrs = 180 days = 6 months
271+
kwargs["cache_hrs"] = 4320
271272
if self.parent_helper.is_url(path):
272273
filename = await self.download(str(path), **kwargs)
273274
if filename is None:

‎bbot/core/modules.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ def __init__(self):
5656
self._shared_deps = dict(SHARED_DEPS)
5757

5858
self.__preloaded = {}
59-
self._modules = {}
6059
self._configs = {}
6160
self.flag_choices = set()
6261
self.all_module_choices = set()
@@ -463,7 +462,6 @@ def load_modules(self, module_names):
463462
for module_name in module_names:
464463
module = self.load_module(module_name)
465464
modules[module_name] = module
466-
self._modules[module_name] = module
467465
return modules
468466

469467
def load_module(self, module_name):

‎bbot/modules/base.py‎

Lines changed: 23 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,14 @@ async def setup(self):
213213

214214
return True
215215

216+
async def setup_deps(self):
217+
"""
218+
Similar to setup(), but reserved for installing dependencies not covered by Ansible.
219+
220+
This should always be used to install static dependencies like AI models, wordlists, etc.
221+
"""
222+
return True
223+
216224
async def handle_event(self, event, **kwargs):
217225
"""Asynchronously handles incoming events that the module is configured to watch.
218226
@@ -620,39 +628,26 @@ def start(self):
620628
name=f"{self.scan.name}.{self.name}._event_handler_watchdog()",
621629
)
622630

623-
async def _setup(self):
624-
"""
625-
Asynchronously sets up the module by invoking its 'setup()' method.
626-
627-
This method catches exceptions during setup, sets the module's error state if necessary, and determines the
628-
status code based on the result of the setup process.
629-
630-
Args:
631-
None
632-
633-
Returns:
634-
tuple: A tuple containing the module's name, status (True for success, False for hard-fail, None for soft-fail),
635-
and an optional status message.
636-
637-
Raises:
638-
Exception: Captured exceptions from the 'setup()' method are logged, but not propagated.
639-
640-
Notes:
641-
- The 'setup()' method can return either a simple boolean status or a tuple of status and message.
642-
- A WordlistError exception triggers a soft-fail status.
643-
- The debug log will contain setup status information for the module.
644-
"""
631+
async def _setup(self, deps_only=False):
632+
""" """
645633
status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}
646634

647635
status = False
648636
self.debug(f"Setting up module {self.name}")
649637
try:
650-
result = await self.setup()
651-
if type(result) == tuple and len(result) == 2:
652-
status, msg = result
653-
else:
654-
status = result
655-
msg = status_codes[status]
638+
funcs = [self.setup_deps]
639+
if not deps_only:
640+
funcs.append(self.setup)
641+
for func in funcs:
642+
self.debug(f"Running {self.name}.{func.__name__}()")
643+
result = await func()
644+
if type(result) == tuple and len(result) == 2:
645+
status, msg = result
646+
else:
647+
status = result
648+
msg = status_codes[status]
649+
if status is False:
650+
break
656651
self.debug(f"Finished setting up module {self.name}")
657652
except Exception as e:
658653
self.set_error_state(f"Unexpected error during module setup: {e}", critical=True)

‎bbot/modules/dnsbrute.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum):
2323
dedup_strategy = "lowest_parent"
2424
_qsize = 10000
2525

26+
async def setup_deps(self):
27+
self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
28+
# tell the dnsbrute helper to fetch the resolver file
29+
await self.helpers.dns.brute.resolver_file()
30+
return True
31+
2632
async def setup(self):
2733
self.max_depth = max(1, self.config.get("max_depth", 5))
28-
self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
2934
self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
3035
self.wordlist_size = len(self.subdomain_list)
3136
return await super().setup()

‎bbot/modules/ffuf.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,15 @@ class ffuf(BaseModule):
3737

3838
in_scope_only = True
3939

40+
async def setup_deps(self):
41+
self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
42+
return True
43+
4044
async def setup(self):
4145
self.proxy = self.scan.web_config.get("http_proxy", "")
4246
self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
4347
wordlist_url = self.config.get("wordlist", "")
4448
self.debug(f"Using wordlist [{wordlist_url}]")
45-
self.wordlist = await self.helpers.wordlist(wordlist_url)
4649
self.wordlist_lines = self.generate_wordlist(self.wordlist)
4750
self.tempfile, tempfile_len = self.generate_templist()
4851
self.rate = self.config.get("rate", 0)

‎bbot/modules/ffuf_shortnames.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,17 @@ def find_common_prefixes(strings, minimum_set_length=4):
8787
found_prefixes.add(prefix)
8888
return list(found_prefixes)
8989

90-
async def setup(self):
91-
self.proxy = self.scan.web_config.get("http_proxy", "")
92-
self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
90+
async def setup_deps(self):
9391
wordlist_extensions = self.config.get("wordlist_extensions", "")
9492
if not wordlist_extensions:
9593
wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
9694
self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
9795
self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
96+
return True
97+
98+
async def setup(self):
99+
self.proxy = self.scan.web_config.get("http_proxy", "")
100+
self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
98101
self.ignore_redirects = self.config.get("ignore_redirects")
99102
self.max_predictions = self.config.get("max_predictions")
100103
self.find_subwords = self.config.get("find_subwords")

0 commit comments

Comments
 (0)