diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index d641828515..9308f218c1 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -5,11 +5,13 @@ from typing import TYPE_CHECKING from urllib.parse import quote, unquote +from dissect.target.filesystem import VirtualFilesystem from dissect.target.filesystems.dir import DirectoryFilesystem from dissect.target.filesystems.zip import ZipFilesystem from dissect.target.helpers.fsutil import basename, dirname, join from dissect.target.loaders.dir import DirLoader, find_dirs, map_dirs from dissect.target.plugin import OperatingSystem +from dissect.target.plugins.apps.edr.velociraptor import VELOCIRAPTOR_RESULTS if TYPE_CHECKING: from pathlib import Path @@ -31,7 +33,11 @@ def find_fs_directories(path: Path) -> tuple[OperatingSystem | None, list[Path] accessor_root = fs_root.joinpath(accessor) if accessor_root.exists(): os_type, dirs = find_dirs(accessor_root) - if os_type in [OperatingSystem.UNIX, OperatingSystem.LINUX, OperatingSystem.OSX]: + if os_type in [ + OperatingSystem.UNIX, + OperatingSystem.LINUX, + OperatingSystem.OSX, + ]: return os_type, [dirs[0]] # Windows @@ -69,7 +75,6 @@ def extract_drive_letter(name: str) -> str | None: # X: in URL encoding if len(name) == 4 and name.endswith("%3A"): return name[0].lower() - return None @@ -137,6 +142,21 @@ def map(self, target: Target) -> None: zipfs=VelociraptorZipFilesystem, ) + if (results := self.root.joinpath("results")).is_dir(): + # Map artifact results collected by Velociraptor + vfs = VirtualFilesystem() + + for artifact in results.iterdir(): + if not artifact.name.endswith(".json"): + continue + + vfs.map_file_fh(artifact.name, artifact.open("rb")) + + if (uploads := self.root.joinpath("uploads.json")).exists(): + vfs.map_file_fh(uploads.name, uploads.open("rb")) + + target.fs.mount(VELOCIRAPTOR_RESULTS, vfs) + class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: diff --git a/dissect/target/plugins/apps/edr/__init__.py b/dissect/target/plugins/apps/edr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dissect/target/plugins/apps/edr/acquire.py b/dissect/target/plugins/apps/edr/acquire.py new file mode 100644 index 0000000000..aa20d1a6b5 --- /dev/null +++ b/dissect/target/plugins/apps/edr/acquire.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import csv +import gzip +from typing import TYPE_CHECKING + +from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.helpers.record import TargetRecordDescriptor +from dissect.target.plugin import Plugin, export + +if TYPE_CHECKING: + from collections.abc import Iterator + + from dissect.target.target import Target + +AcquireOpenHandlesRecord = TargetRecordDescriptor( + "filesystem/acquire_open_handles", + [ + ("path", "name"), + ("string", "handle_type"), + ("string", "object"), + ("varint", "unique_process_id"), + ("varint", "handle_value"), + ("varint", "granted_access"), + ("varint", "creator_back_trace_index"), + ("varint", "object_type_index"), + ("varint", "handle_attributes"), + ("varint", "reserved"), + ], +) + +AcquireHashRecord = TargetRecordDescriptor( + "filesystem/acquire_hash", + [ + ("path", "path"), + ("filesize", "filesize"), + ("digest", "digest"), + ], +) + + +class AcquirePlugin(Plugin): + """Returns records from data collected by Acquire.""" + + __namespace__ = "acquire" + + def __init__(self, target: Target): + super().__init__(target) + self.hash_file = target.fs.path("$metadata$/file-hashes.csv.gz") + self.open_handles_file = target.fs.path("$metadata$/open_handles.csv.gz") + + def check_compatible(self) -> None: + if not self.hash_file.exists() and not self.open_handles_file.exists(): + raise UnsupportedPluginError("No hash file or open handles found") + + @export(record=AcquireHashRecord) + def hashes(self) -> Iterator[AcquireHashRecord]: + """Return file hashes collected by Acquire. + + An Acquire file container contains a file hashes csv when the hashes module was used. The content of this csv + file is returned. + """ + if self.hash_file.exists(): + with self.hash_file.open() as fh, gzip.open(fh, "rt") as gz_fh: + for row in csv.DictReader(gz_fh): + yield AcquireHashRecord( + path=self.target.fs.path(row["path"]), + filesize=row["file-size"], + digest=(row["md5"] or None, row["sha1"] or None, row["sha256"] or None), + _target=self.target, + ) + + @export(record=AcquireOpenHandlesRecord) + def handles(self) -> Iterator[AcquireOpenHandlesRecord]: + """Return open handles collected by Acquire. + + An Acquire file container contains an open handles csv when the handles module was used. The content of this csv + file is returned. + """ + if self.open_handles_file.exists(): + with self.open_handles_file.open() as fh, gzip.open(fh, "rt") as gz_fh: + for row in csv.DictReader(gz_fh): + if name := row.get("name"): + row.update({"name": self.target.fs.path(name)}) + yield AcquireOpenHandlesRecord(**row, _target=self.target) diff --git a/dissect/target/plugins/apps/edr/velociraptor.py b/dissect/target/plugins/apps/edr/velociraptor.py new file mode 100644 index 0000000000..0638040a3b --- /dev/null +++ b/dissect/target/plugins/apps/edr/velociraptor.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +import re +import urllib.parse +from functools import lru_cache +from typing import TYPE_CHECKING + +from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.helpers.record import DynamicDescriptor, TargetRecordDescriptor +from dissect.target.plugin import Plugin, export + +if TYPE_CHECKING: + from collections.abc import Iterator + + from flow.record import Record + + from dissect.target.target import Target + +VELOCIRAPTOR_RESULTS = "/$velociraptor_results$" +ISO_8601_PATTERN = r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?" + + +class VelociraptorRecordBuilder: + def __init__(self, artifact_name: str): + self._create_event_descriptor = lru_cache(4096)(self._create_event_descriptor) + self.record_name = f"velociraptor/{artifact_name}" + + def build(self, object: dict, target: Target) -> TargetRecordDescriptor: + """Builds a Velociraptor record.""" + record_values = {} + record_fields = [] + + record_values["_target"] = target + + for key, value in object.items(): + # Reserved by flow.record + if key.startswith("_"): + continue + + key = key.lower().replace("(", "_").replace(")", "_") + + if re.match(ISO_8601_PATTERN, str(value)): + record_type = "datetime" + elif isinstance(value, list): + record_type = "string[]" + elif isinstance(value, int): + record_type = "varint" + elif key == "hash": + record_type = "digest" + value = (value.get("MD5"), value.get("SHA1"), value.get("SHA256")) + elif isinstance(value, str): + record_type = "string" + elif isinstance(value, dict): + record_type = "record" + value = self.build(value, target) + else: + record_type = "dynamic" + + record_fields.append((record_type, key)) + record_values[key] = value + + # tuple conversion here is needed for lru_cache + desc = self._create_event_descriptor(tuple(record_fields)) + return desc(**record_values) + + def _create_event_descriptor(self, record_fields: list[tuple[str, str]]) -> TargetRecordDescriptor: + return TargetRecordDescriptor(self.record_name, record_fields) + + +class VelociraptorPlugin(Plugin): + """Returns records from Velociraptor artifacts.""" + + __namespace__ = "velociraptor" + + def __init__(self, target: Target): + super().__init__(target) + self.results_dir = target.fs.path(VELOCIRAPTOR_RESULTS) + + def check_compatible(self) -> None: + if not self.results_dir.exists(): + raise UnsupportedPluginError("No Velociraptor artifacts found") + + @export(record=DynamicDescriptor(["datetime"])) + def results(self) -> Iterator[Record]: + """Return Rapid7 Velociraptor artifacts. + + References: + - https://docs.velociraptor.app/docs/vql/artifacts/ + """ + for artifact in self.results_dir.glob("*.json"): + # "Windows.KapeFiles.Targets%2FAll\ File\ Metadata.json" becomes "windows_kapefiles_targets" + artifact_name = ( + urllib.parse.unquote(artifact.name.removesuffix(".json")).split("/")[0].lower().replace(".", "_") + ) + record_builder = VelociraptorRecordBuilder(artifact_name) + + for line in artifact.open("rt"): + if not (line := line.strip()): + continue + + try: + object = json.loads(line) + yield record_builder.build(object, self.target) + except json.decoder.JSONDecodeError: + self.target.log.warning( + "Could not decode Velociraptor JSON log line in file %s: %s", + artifact, + line, + ) + continue diff --git a/dissect/target/plugins/filesystem/acquire_handles.py b/dissect/target/plugins/filesystem/acquire_handles.py deleted file mode 100644 index 76d4b64795..0000000000 --- a/dissect/target/plugins/filesystem/acquire_handles.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import csv -import gzip -from typing import TYPE_CHECKING - -from dissect.target.exceptions import UnsupportedPluginError -from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export - -if TYPE_CHECKING: - from collections.abc import Iterator - - from dissect.target.target import Target - -AcquireOpenHandlesRecord = TargetRecordDescriptor( - "filesystem/acquire_open_handles", - [ - ("path", "name"), - ("string", "handle_type"), - ("string", "object"), - ("varint", "unique_process_id"), - ("varint", "handle_value"), - ("varint", "granted_access"), - ("varint", "creator_back_trace_index"), - ("varint", "object_type_index"), - ("varint", "handle_attributes"), - ("varint", "reserved"), - ], -) - - -class OpenHandlesPlugin(Plugin): - """Plugin to return open file handles collected by Acquire.""" - - def __init__(self, target: Target): - super().__init__(target) - self.open_handles_file = target.fs.path("$metadata$/open_handles.csv.gz") - - def check_compatible(self) -> None: - if not self.open_handles_file.exists(): - raise UnsupportedPluginError("No open handles found") - - @export(record=AcquireOpenHandlesRecord) - def acquire_handles(self) -> Iterator[AcquireOpenHandlesRecord]: - """Return open handles collected by Acquire. - - An Acquire file container contains an open handles csv when the handles module was used. The content of this csv - file is returned. - """ - with self.open_handles_file.open() as fh, gzip.open(fh, "rt") as gz_fh: - for row in csv.DictReader(gz_fh): - if name := row.get("name"): - row.update({"name": self.target.fs.path(name)}) - yield AcquireOpenHandlesRecord(_target=self.target, **row) diff --git a/dissect/target/plugins/filesystem/acquire_hash.py b/dissect/target/plugins/filesystem/acquire_hash.py deleted file mode 100644 index 8986beb9c0..0000000000 --- a/dissect/target/plugins/filesystem/acquire_hash.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import csv -import gzip -from typing import TYPE_CHECKING - -from dissect.target.exceptions import UnsupportedPluginError -from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export - -if TYPE_CHECKING: - from collections.abc import Iterator - - from dissect.target.target import Target - -AcquireHashRecord = TargetRecordDescriptor( - "filesystem/acquire_hash", - [ - ("path", "path"), - ("filesize", "filesize"), - ("digest", "digest"), - ], -) - - -class AcquireHashPlugin(Plugin): - """Plugin to return file hashes collected by Acquire.""" - - def __init__(self, target: Target): - super().__init__(target) - self.hash_file = target.fs.path("$metadata$/file-hashes.csv.gz") - - def check_compatible(self) -> None: - if not self.hash_file.exists(): - raise UnsupportedPluginError("No hash file found") - - @export(record=AcquireHashRecord) - def acquire_hashes(self) -> Iterator[AcquireHashRecord]: - """Return file hashes collected by Acquire. - - An Acquire file container contains a file hashes csv when the hashes module was used. The content of this csv - file is returned. - """ - - with self.hash_file.open() as fh, gzip.open(fh, "rt") as gz_fh: - for row in csv.DictReader(gz_fh): - yield AcquireHashRecord( - path=self.target.fs.path(row["path"]), - filesize=row["file-size"], - digest=(row["md5"] or None, row["sha1"] or None, row["sha256"] or None), - _target=self.target, - ) diff --git a/tests/_data/plugins/filesystem/acquire_handles/test-acquire-handles.tar b/tests/_data/plugins/apps/edr/acquire/handles/test-acquire-handles.tar similarity index 100% rename from tests/_data/plugins/filesystem/acquire_handles/test-acquire-handles.tar rename to tests/_data/plugins/apps/edr/acquire/handles/test-acquire-handles.tar diff --git a/tests/_data/plugins/filesystem/acquire_hash/test-acquire-hash.tar b/tests/_data/plugins/apps/edr/acquire/hash/test-acquire-hash.tar similarity index 100% rename from tests/_data/plugins/filesystem/acquire_hash/test-acquire-hash.tar rename to tests/_data/plugins/apps/edr/acquire/hash/test-acquire-hash.tar diff --git a/tests/_data/plugins/apps/edr/velociraptor/Windows.Memory.ProcessInfo.json b/tests/_data/plugins/apps/edr/velociraptor/Windows.Memory.ProcessInfo.json new file mode 100644 index 0000000000..085c99b800 --- /dev/null +++ b/tests/_data/plugins/apps/edr/velociraptor/Windows.Memory.ProcessInfo.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cb561eb0d37ac2b84cbb3afef20685b1fe3524c46a9b59f547ac52019aafc3 +size 4211 diff --git a/tests/_data/plugins/apps/edr/velociraptor/windows-uploads.json b/tests/_data/plugins/apps/edr/velociraptor/windows-uploads.json new file mode 100644 index 0000000000..5c35e35ae9 --- /dev/null +++ b/tests/_data/plugins/apps/edr/velociraptor/windows-uploads.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15668b953b1c69d41917c78f1e351b81c3e9a0112952135f8e5e2353b43ea6b7 +size 1782 diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 6b1d4c09a4..b7c84cc07c 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -24,6 +24,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/$Extend", f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/windows/system32", f"uploads/{sub_dir}/%5C%5C.%5CC%3A/%2ETEST", + "results", ] root = tmp_path mkdirs(root, paths) @@ -52,16 +53,10 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: @pytest.mark.parametrize( - ("sub_dir", "other_dir"), - [ - ("mft", "auto"), - ("ntfs", "auto"), - ("ntfs_vss", "auto"), - ("lazy_ntfs", "auto"), - ("auto", "ntfs"), - ], + "sub_dir", + ["mft", "ntfs", "ntfs_vss", "lazy_ntfs", "auto"], ) -def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_path: Path) -> None: +def test_windows_ntfs(sub_dir: str, target_bare: Target, tmp_path: Path) -> None: root = create_root(sub_dir, tmp_path) assert VelociraptorLoader.detect(root) is True diff --git a/tests/plugins/apps/edr/__init__.py b/tests/plugins/apps/edr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plugins/apps/edr/test_acquire.py b/tests/plugins/apps/edr/test_acquire.py new file mode 100644 index 0000000000..3ab863f48b --- /dev/null +++ b/tests/plugins/apps/edr/test_acquire.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dissect.target.plugins.apps.edr.acquire import AcquirePlugin +from dissect.target.target import Target +from tests._utils import absolute_path + + +def test_acquire_handles_plugin() -> None: + file_hashes_target = Target().open(absolute_path("_data/plugins/apps/edr/acquire/handles/test-acquire-handles.tar")) + file_hashes_target.add_plugin(AcquirePlugin) + + results = list(file_hashes_target.acquire.handles()) + first_result = results[0] + + assert first_result.name == r"\Windows\Fonts" + assert first_result.handle_type == "EtwRegistration" + assert first_result.unique_process_id == 1 + assert first_result.object == "0xfffftest" + assert results[-1].unique_process_id == 124 + assert len(results) == 124 + + +def test_acquire_hash_plugin() -> None: + file_hashes_target = Target().open(absolute_path("_data/plugins/apps/edr/acquire/hash/test-acquire-hash.tar")) + file_hashes_target.add_plugin(AcquirePlugin) + + results = list(file_hashes_target.acquire.hashes()) + + assert results[0].path == "/sysvol/Windows/bfsvc.exe" + assert len(results) == 998 diff --git a/tests/plugins/apps/edr/test_velociraptor.py b/tests/plugins/apps/edr/test_velociraptor.py new file mode 100644 index 0000000000..7f2522e9fe --- /dev/null +++ b/tests/plugins/apps/edr/test_velociraptor.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dissect.target.loaders.velociraptor import VelociraptorLoader +from dissect.target.plugins.apps.edr.velociraptor import VelociraptorPlugin +from tests._utils import absolute_path +from tests.loaders.test_velociraptor import create_root + +if TYPE_CHECKING: + from pathlib import Path + + from dissect.target.target import Target + + +def test_windows_velociraptor(target_win: Target, tmp_path: Path) -> None: + """Test that a Windows Velociraptor artefact result is correctly parsed.""" + root = create_root("ntfs", tmp_path) + + with absolute_path("_data/plugins/apps/edr/velociraptor/windows-uploads.json").open("rb") as fh: + root.joinpath("uploads.json").write_bytes(fh.read()) + + with absolute_path("_data/plugins/apps/edr/velociraptor/Windows.Memory.ProcessInfo.json").open("rb") as fh: + root.joinpath("results/Windows.Memory.ProcessInfo.json").write_bytes(fh.read()) + + assert VelociraptorLoader.detect(root) is True + + loader = VelociraptorLoader(root) + loader.map(target_win) + target_win.apply() + + target_win.add_plugin(VelociraptorPlugin) + + results = list(target_win.velociraptor()) + + record = results[0] + + assert record.name == "Microsoft.SharePoint.exe" + assert record.pebbaseaddress == "0x295000" + assert record.pid == 8120 + assert ( + record.imagepathname + == "C:\\Users\\IEUser\\AppData\\Local\\Microsoft\\OneDrive\\24.070.0407.0003\\Microsoft.SharePoint.exe" + ) + assert record.commandline == "/silentConfig" + assert record.currentdirectory == "C:\\Windows\\system32\\" + assert record._desc.name == "velociraptor/windows_memory_processinfo" + assert record.env.allusersprofile == "C:\\ProgramData" diff --git a/tests/plugins/filesystem/test_acquire_handles.py b/tests/plugins/filesystem/test_acquire_handles.py deleted file mode 100644 index 35b6164add..0000000000 --- a/tests/plugins/filesystem/test_acquire_handles.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from dissect.target.plugins.filesystem.acquire_handles import OpenHandlesPlugin -from dissect.target.target import Target -from tests._utils import absolute_path - - -def test_acquire_handles_plugin() -> None: - file_hashes_target = Target().open( - absolute_path("_data/plugins/filesystem/acquire_handles/test-acquire-handles.tar") - ) - file_hashes_target.add_plugin(OpenHandlesPlugin) - - results = list(file_hashes_target.acquire_handles()) - first_result = results[0] - - assert first_result.name == r"\Windows\Fonts" - assert first_result.handle_type == "EtwRegistration" - assert first_result.unique_process_id == 1 - assert first_result.object == "0xfffftest" - assert results[-1].unique_process_id == 124 - assert len(results) == 124 diff --git a/tests/plugins/filesystem/test_acquire_hash.py b/tests/plugins/filesystem/test_acquire_hash.py deleted file mode 100644 index 305d6cba46..0000000000 --- a/tests/plugins/filesystem/test_acquire_hash.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from dissect.target.plugins.filesystem.acquire_hash import AcquireHashPlugin -from dissect.target.target import Target -from tests._utils import absolute_path - - -def test_acquire_hash_plugin() -> None: - file_hashes_target = Target().open(absolute_path("_data/plugins/filesystem/acquire_hash/test-acquire-hash.tar")) - file_hashes_target.add_plugin(AcquireHashPlugin) - - results = list(file_hashes_target.acquire_hashes()) - - assert results[0].path == "/sysvol/Windows/bfsvc.exe" - assert len(results) == 998