Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
130 commits
Select commit Hold shift + click to select a range
fefc52a
waf_bypass initial
liquidsec Aug 1, 2025
466a2ad
fixing source event bug
liquidsec Aug 4, 2025
7049bff
continued development
liquidsec Aug 6, 2025
2041483
Merge branch 'dev' into waf-bypass
liquidsec Aug 6, 2025
f68de32
adding asn helper test, fixing report
liquidsec Aug 6, 2025
b1e175a
normalizing nomenclature, adding tests
liquidsec Aug 7, 2025
3cd55a6
add meta
liquidsec Aug 11, 2025
c228095
Merge branch 'dev' into waf-bypass
liquidsec Aug 11, 2025
163ea09
output modifications
liquidsec Aug 15, 2025
dee9079
fix debug messages and add ip filter to ip check
liquidsec Aug 15, 2025
9626f28
add preset
liquidsec Aug 15, 2025
cae129d
fix bug when no subnets in asn
liquidsec Aug 18, 2025
53ea0b3
virtualhost overhaul initial
liquidsec Aug 27, 2025
3bf032a
tweak
liquidsec Aug 27, 2025
38d4d2e
virtualhost update major refactor
liquidsec Aug 30, 2025
454d7d1
virtual host another major refactor
liquidsec Sep 2, 2025
f04c486
changes to as_completed
liquidsec Sep 2, 2025
1d16b49
pinned curl
liquidsec Sep 2, 2025
7c32cb1
fix missing arg
liquidsec Sep 2, 2025
3447866
finish rename
liquidsec Sep 2, 2025
0d98531
small refactor
liquidsec Sep 2, 2025
eb93201
fix async handling
liquidsec Sep 3, 2025
84c0f8e
presets and other adjustments
liquidsec Sep 3, 2025
ef1674f
change as_completed to only accept coros
liquidsec Sep 3, 2025
bd51281
starting to back out of debug, more tweaks
liquidsec Sep 4, 2025
d160941
special virtual host only ignore strings
liquidsec Sep 4, 2025
d19f324
add module test
liquidsec Sep 4, 2025
a5847c5
Merge branch 'dev' into virtualhost-upgrade
liquidsec Sep 4, 2025
ba9fcae
typo
liquidsec Sep 4, 2025
951ca41
fix virtual_host event test fixtures
liquidsec Sep 4, 2025
9940355
lint
liquidsec Sep 4, 2025
525481e
yanking debug stuff, polishing
liquidsec Sep 4, 2025
b3757ec
make curl command message debug only
liquidsec Sep 4, 2025
3082880
Merge branch 'dev' into virtualhost-upgrade
liquidsec Sep 4, 2025
f160f00
finish() scan bug fix
liquidsec Sep 4, 2025
978b5cd
go mariners
liquidsec Sep 5, 2025
7def332
as_completed error handling, tests
liquidsec Sep 5, 2025
2ba8318
yet another major refactor
liquidsec Sep 5, 2025
824d35d
i let the clanker try to alphabetize
liquidsec Sep 5, 2025
9fdf579
lint
liquidsec Sep 5, 2025
53512df
fixing test
liquidsec Sep 6, 2025
fbeadc0
adjustments
liquidsec Sep 7, 2025
cbe372d
more refactoring
liquidsec Sep 8, 2025
716652e
error message adjustments
liquidsec Sep 8, 2025
bf978b9
error handling
liquidsec Sep 9, 2025
05505d8
add web.response_similarity web helper
liquidsec Sep 10, 2025
7a24d16
use new response_similarity helper
liquidsec Sep 10, 2025
7f4c445
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Sep 10, 2025
c926393
initial change from merge
liquidsec Sep 10, 2025
6d4c88b
further generalizing comparison helper
liquidsec Sep 10, 2025
f957f5e
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Sep 10, 2025
41aa18c
real asn db url
liquidsec Sep 11, 2025
b147ae5
rework comparison logic to use helper
liquidsec Sep 11, 2025
2ed2296
correct ASN expansion, bug fixes, tests
liquidsec Sep 19, 2025
fafdad1
bugfix
liquidsec Sep 19, 2025
b6aa2a5
fixing tests
liquidsec Sep 19, 2025
bff0b7e
lint
liquidsec Sep 19, 2025
2aa0685
fixing bug
liquidsec Sep 20, 2025
5f01007
fix tests
liquidsec Sep 20, 2025
d5fbae3
fix more tests
liquidsec Sep 20, 2025
30624c6
fixing test
liquidsec Sep 20, 2025
ce16a80
more test fixes, adjustments to prep
liquidsec Sep 20, 2025
4f65a46
fixing slop
liquidsec Sep 22, 2025
d8a5a64
more reworking around the prep() change
liquidsec Sep 22, 2025
3507d00
more test fixing
liquidsec Sep 22, 2025
06b8de5
more test fixes
liquidsec Sep 22, 2025
32efd2e
rate-limit use retry-after
liquidsec Sep 22, 2025
0c43022
more test fixes
liquidsec Sep 22, 2025
ee844c6
use correct method
liquidsec Sep 22, 2025
1b12214
change to new function name
liquidsec Sep 22, 2025
6effa6b
fix asn helper access
liquidsec Sep 22, 2025
80789b0
yet more test fixes
liquidsec Sep 22, 2025
c0f9467
Merge branch 'dev' into virtualhost-upgrade
liquidsec Sep 22, 2025
7a2bc99
yet even still more test fixes
liquidsec Sep 22, 2025
d483997
slightly change waf_bypass detection criteria
liquidsec Sep 22, 2025
8915855
massive test fixes (from prep() changes)
liquidsec Sep 23, 2025
183a6f0
yet even more test fixes still
liquidsec Sep 23, 2025
2d5498e
ruff format
liquidsec Sep 23, 2025
9555205
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Sep 23, 2025
ee03e61
fix even yet more additional tests
liquidsec Sep 23, 2025
aab237a
ugggggggggggggggggggggggggggggg tests
liquidsec Sep 23, 2025
11710ea
test initialize order
liquidsec Sep 23, 2025
40a35c1
another test fix
liquidsec Sep 23, 2025
86b3f03
more test stuff again the sequel
liquidsec Sep 23, 2025
20ef830
even more :(
liquidsec Sep 24, 2025
391155f
test fix
liquidsec Sep 24, 2025
3dec03a
asdf
liquidsec Sep 24, 2025
d455b7c
better large ip_range behavior
liquidsec Sep 24, 2025
df66c25
fixing unknown asn system
liquidsec Sep 25, 2025
eb7f570
change to simhash for comparison
liquidsec Sep 26, 2025
33ef6db
remove debug messages
liquidsec Sep 26, 2025
9576232
debug junk
liquidsec Sep 26, 2025
323284e
add simhash helper
liquidsec Sep 26, 2025
d3a68d5
fixing inefficient seed event class sorting
liquidsec Sep 26, 2025
39bc9fe
remove unnecessary warning
liquidsec Sep 26, 2025
ddc0a9c
DRY
liquidsec Sep 26, 2025
3faf36f
refactor for correct api format
liquidsec Sep 26, 2025
bdce02d
LRU cache, fix rate limiting
liquidsec Sep 26, 2025
db741fb
Merge branch 'waf-bypass' into better-text-compare
liquidsec Sep 26, 2025
5a1a66d
remove debug
liquidsec Sep 26, 2025
a66b012
comments
TheTechromancer Sep 26, 2025
4da27b8
remove per host only
liquidsec Sep 26, 2025
12bf1f9
simhash helper update
liquidsec Sep 27, 2025
f446117
major waf_bypass refactor
liquidsec Sep 27, 2025
321c259
format
liquidsec Sep 27, 2025
6231157
Merge branch 'waf-bypass' into better-text-compare
liquidsec Sep 27, 2025
35bc4b7
Merge pull request #2713 from blacklanternsecurity/better-text-compare
liquidsec Sep 27, 2025
49e724b
Merge branch 'dev' into virtualhost-upgrade
liquidsec Sep 29, 2025
0173e2a
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Sep 29, 2025
cd0c3f9
Merge branch 'dev' into virtualhost-upgrade
liquidsec Oct 9, 2025
635aa4d
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Oct 9, 2025
0cc459d
add virtualhost http_response emittal
liquidsec Oct 15, 2025
ca64515
fix header extraction curl helper
liquidsec Oct 16, 2025
7bb3989
Merge pull request #2738 from blacklanternsecurity/virtualhost-httpre…
liquidsec Oct 16, 2025
15e5eb5
Merge branch 'dev' into virtualhost-upgrade
liquidsec Oct 16, 2025
cf9ae83
Merge branch 'virtualhost-upgrade' into waf-bypass
liquidsec Oct 16, 2025
02129b0
remove unnecessary cache
liquidsec Oct 16, 2025
5147d68
forklifting asn human output code
liquidsec Oct 16, 2025
2863934
clean up duplicate
liquidsec Oct 16, 2025
5cd05d0
more efficient asn report (using helper)
liquidsec Oct 16, 2025
1de5db2
some cleanup
liquidsec Oct 17, 2025
3e03865
Merge pull request #2553 from blacklanternsecurity/waf-bypass
liquidsec Oct 17, 2025
2191c72
temporarility removing new modules
liquidsec Oct 17, 2025
2686341
temporarily remove presets
liquidsec Oct 17, 2025
31c3e12
temp removal
liquidsec Oct 17, 2025
56f5d53
ruff format
liquidsec Oct 17, 2025
1740a28
temporarily revert
liquidsec Oct 17, 2025
04206e2
fix test again
liquidsec Oct 17, 2025
96658b6
fixing test
liquidsec Oct 17, 2025
6ce1295
temp removal
liquidsec Oct 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions bbot/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,13 @@ async def _main():
import traceback
from contextlib import suppress

# fix tee buffering
sys.stdout.reconfigure(line_buffering=True)
# fix tee buffering (only if on real TTY)
if hasattr(sys.stdout, "reconfigure"):
try:
if sys.stdout.isatty():
sys.stdout.reconfigure(line_buffering=True)
except Exception:
pass

log = logging.getLogger("bbot.cli")

Expand Down Expand Up @@ -90,7 +95,7 @@ async def _main():
preset._default_output_modules = options.output_modules
preset._default_internal_modules = []

preset.bake()
await preset.bake()

# --list-modules
if options.list_modules:
Expand Down Expand Up @@ -150,6 +155,8 @@ async def _main():
log.warning(str(e))
return

await scan._prep()

deadly_modules = [
m for m in scan.preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", [])
]
Expand Down Expand Up @@ -191,12 +198,18 @@ async def _main():
log.verbose(row)

scan.helpers.word_cloud.load()
await scan._prep()

if not options.dry_run:
log.trace(f"Command: {' '.join(sys.argv)}")

if sys.stdin.isatty():
try:
is_tty = (
hasattr(sys.stdin, "isatty") and not getattr(sys.stdin, "closed", False) and sys.stdin.isatty()
)
except Exception:
is_tty = False

if is_tty:
# warn if any targets belong directly to a cloud provider
if not scan.preset.strict_scope:
for event in scan.target.seeds.event_seeds:
Expand Down
49 changes: 36 additions & 13 deletions bbot/core/event/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,7 @@ def parent(self, parent):
self.web_spider_distance = getattr(parent, "web_spider_distance", 0)
event_has_url = getattr(self, "parsed_url", None) is not None
for t in parent.tags:
if t in ("affiliate",):
if t in ("affiliate"):
self.add_tag(t)
elif t.startswith("mutation-"):
self.add_tag(t)
Expand Down Expand Up @@ -1135,6 +1135,41 @@ class ASN(DictEvent):
_always_emit = True
_quick_emit = True

def sanitize_data(self, data):
if not isinstance(data, int):
raise ValidationError(f"ASN number must be an integer: {data}")
return data

def _data_human(self):
"""Create a concise human-readable representation of ASN data."""
# Start with basic ASN info
display_data = {"asn": str(self.data)}

# Try to get additional ASN data from the helper if available
if hasattr(self, "scan") and self.scan and hasattr(self.scan, "helpers"):
try:
# Check if we can access the ASN helper synchronously
asn_helper = self.scan.helpers.asn
# Try to get cached data first (this should be synchronous)
cached_data = asn_helper._cache_lookup_asn(self.data)
if cached_data:
display_data.update(
{
"name": cached_data.get("name", ""),
"description": cached_data.get("description", ""),
"country": cached_data.get("country", ""),
}
)
# Replace subnets list with count for readability
subnets = cached_data.get("subnets", [])
if subnets and isinstance(subnets, list):
display_data["subnet_count"] = len(subnets)
except Exception:
# If anything fails, just return basic ASN info
pass

return json.dumps(display_data, sort_keys=True)


class CODE_REPOSITORY(DictHostEvent):
_always_emit = True
Expand Down Expand Up @@ -1604,18 +1639,6 @@ def _pretty_string(self):
return self.data["technology"]


class VHOST(DictHostEvent):
class _data_validator(BaseModel):
host: str
vhost: str
url: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _pretty_string(self):
return self.data["vhost"]


class PROTOCOL(DictHostEvent):
class _data_validator(BaseModel):
host: str
Expand Down
71 changes: 69 additions & 2 deletions bbot/core/event/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,20 @@
bbot_event_seeds = {}


# Pre-compute sorted event classes for performance
# This is computed once when the module is loaded instead of on every EventSeed() call
def _get_sorted_event_classes():
"""
Sort event classes by priority (higher priority first).
This ensures specific patterns like ASN:12345 are checked before broad patterns like hostname:port.
"""
return sorted(bbot_event_seeds.items(), key=lambda x: getattr(x[1], "priority", 5), reverse=True)


# This will be populated after all event seed classes are registered
_sorted_event_classes = None


"""
An "Event Seed" is a lightweight event containing only the minimum logic required to:
- parse input to determine the event type + data
Expand All @@ -18,6 +32,19 @@
It's useful for quickly parsing target lists without the cpu+memory overhead of creating full-fledged BBOT events

Not every type of BBOT event needs to be represented here. Only ones that are meant to be targets.

PRIORITY SYSTEM:
Event seeds support a priority system to control the order in which regex patterns are checked.
This prevents conflicts where one event type's regex might incorrectly match another type's input.

Priority values:
- Higher numbers = checked first
- Default priority = 5
- Range: 1-10

To set priority on an event seed class:
class MyEventSeed(BaseEventSeed):
priority = 8 # Higher than default, will be checked before most others
"""


Expand All @@ -27,17 +54,25 @@ class EventSeedRegistry(type):
"""

def __new__(mcs, name, bases, attrs):
global bbot_event_seeds
global bbot_event_seeds, _sorted_event_classes
cls = super().__new__(mcs, name, bases, attrs)
# Don't register the base EventSeed class
if name != "BaseEventSeed":
bbot_event_seeds[cls.__name__] = cls
# Recompute sorted classes whenever a new event seed is registered
_sorted_event_classes = _get_sorted_event_classes()
return cls


def EventSeed(input):
input = smart_encode_punycode(smart_decode(input).strip())
for _, event_class in bbot_event_seeds.items():

# Use pre-computed sorted event classes for better performance
global _sorted_event_classes
if _sorted_event_classes is None:
_sorted_event_classes = _get_sorted_event_classes()

for _, event_class in _sorted_event_classes:
if hasattr(event_class, "precheck"):
if event_class.precheck(input):
return event_class(input)
Expand All @@ -53,6 +88,7 @@ def EventSeed(input):
class BaseEventSeed(metaclass=EventSeedRegistry):
regexes = []
_target_type = "TARGET"
priority = 5 # Default priority for event seed matching (1-10, higher = checked first)

__slots__ = ["data", "host", "port", "input"]

Expand All @@ -76,6 +112,9 @@ def _sanitize_and_extract_host(self, data):
"""
return data, None, None

async def _generate_children(self, helpers):
return []

def _override_input(self, input):
return self.data

Expand Down Expand Up @@ -143,6 +182,7 @@ def _sanitize_and_extract_host(data):

class OPEN_TCP_PORT(BaseEventSeed):
regexes = regexes.event_type_regexes["OPEN_TCP_PORT"]
priority = 1 # Low priority: broad hostname:port pattern should be checked after specific patterns

@staticmethod
def _sanitize_and_extract_host(data):
Expand Down Expand Up @@ -236,3 +276,30 @@ def _override_input(self, input):
@staticmethod
def handle_match(match):
return match.group(1)


class ASN(BaseEventSeed):
regexes = (re.compile(r"^(?:ASN|AS):?(\d+)$", re.I),) # adjust regex to match ASN:17178 AS17178
priority = 10 # High priority

def _override_input(self, input):
return f"ASN:{self.data}"

# ASNs are essentially just a superset of IP_RANGES.
# This method resolves the ASN to a list of IP_RANGES using the ASN API, and then adds the cidr string as a child event seed.
# These will later be automatically resolved to an IP_RANGE event seed and added to the target.
async def _generate_children(self, helpers):
asn_data = await helpers.asn.asn_to_subnets(int(self.data))
children = []
if asn_data:
subnets = asn_data.get("subnets")
if isinstance(subnets, str):
subnets = [subnets]
if subnets:
for cidr in subnets:
children.append(cidr)
return children

@staticmethod
def handle_match(match):
return match.group(1)
Loading
Loading