Add Rapid7 Velociraptor artifacts plugin

dissect/target/loaders/velociraptor.py

@@ -5,11 +5,13 @@
 from typing import TYPE_CHECKING
 from urllib.parse import quote, unquote
 
+from dissect.target.filesystem import VirtualFilesystem
 from dissect.target.filesystems.dir import DirectoryFilesystem
 from dissect.target.filesystems.zip import ZipFilesystem
 from dissect.target.helpers.fsutil import basename, dirname, join
 from dissect.target.loaders.dir import DirLoader, find_dirs, map_dirs
 from dissect.target.plugin import OperatingSystem
+from dissect.target.plugins.apps.edr.velociraptor import VELOCIRAPTOR_RESULTS
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -31,7 +33,11 @@
         accessor_root = fs_root.joinpath(accessor)
         if accessor_root.exists():
             os_type, dirs = find_dirs(accessor_root)
-            if os_type in [OperatingSystem.UNIX, OperatingSystem.LINUX, OperatingSystem.OSX]:
+            if os_type in [
+                OperatingSystem.UNIX,
+                OperatingSystem.LINUX,
+                OperatingSystem.OSX,
+            ]:
                 return os_type, [dirs[0]]
 
     # Windows
@@ -69,7 +75,6 @@
     # X: in URL encoding
     if len(name) == 4 and name.endswith("%3A"):
         return name[0].lower()
-
     return None
 
 
@@ -137,6 +142,21 @@
             zipfs=VelociraptorZipFilesystem,
         )
 
+        if (results := self.root.joinpath("results")).is_dir():
+            # Map artifact results collected by Velociraptor
+            vfs = VirtualFilesystem()
+
+            for artifact in results.iterdir():
+                if not artifact.name.endswith(".json"):
+                    continue
+
+                vfs.map_file_fh(artifact.name, artifact.open("rb"))
+
+            if (uploads := self.root.joinpath("uploads.json")).exists():
+                vfs.map_file_fh(uploads.name, uploads.open("rb"))
+
+            target.fs.mount(VELOCIRAPTOR_RESULTS, vfs)
+
 
 class VelociraptorDirectoryFilesystem(DirectoryFilesystem):
     def _resolve_path(self, path: str) -> Path:

dissect/target/plugins/apps/edr/acquire.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import csv
+import gzip
+from typing import TYPE_CHECKING
+
+from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.helpers.record import TargetRecordDescriptor
+from dissect.target.plugin import Plugin, export
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from dissect.target.target import Target
+
+AcquireOpenHandlesRecord = TargetRecordDescriptor(
+    "filesystem/acquire_open_handles",
+    [
+        ("path", "name"),
+        ("string", "handle_type"),
+        ("string", "object"),
+        ("varint", "unique_process_id"),
+        ("varint", "handle_value"),
+        ("varint", "granted_access"),
+        ("varint", "creator_back_trace_index"),
+        ("varint", "object_type_index"),
+        ("varint", "handle_attributes"),
+        ("varint", "reserved"),
+    ],
+)
+
+AcquireHashRecord = TargetRecordDescriptor(
+    "filesystem/acquire_hash",
+    [
+        ("path", "path"),
+        ("filesize", "filesize"),
+        ("digest", "digest"),
+    ],
+)
+
+
+class AcquirePlugin(Plugin):
+    """Returns records from data collected by Acquire."""
+
+    __namespace__ = "acquire"
+
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.hash_file = target.fs.path("$metadata$/file-hashes.csv.gz")
+        self.open_handles_file = target.fs.path("$metadata$/open_handles.csv.gz")
+
+    def check_compatible(self) -> None:
+        if not self.hash_file.exists() and not self.open_handles_file.exists():
+            raise UnsupportedPluginError("No hash file or open handles found")
+
+    @export(record=AcquireHashRecord)
+    def hashes(self) -> Iterator[AcquireHashRecord]:
+        """Return file hashes collected by Acquire.
+
+        An Acquire file container contains a file hashes csv when the hashes module was used. The content of this csv
+        file is returned.
+        """
+        if self.hash_file.exists():
+            with self.hash_file.open() as fh, gzip.open(fh, "rt") as gz_fh:
+                for row in csv.DictReader(gz_fh):
+                    yield AcquireHashRecord(
+                        path=self.target.fs.path(row["path"]),
+                        filesize=row["file-size"],
+                        digest=(row["md5"] or None, row["sha1"] or None, row["sha256"] or None),
+                        _target=self.target,
+                    )
+
+    @export(record=AcquireOpenHandlesRecord)
+    def handles(self) -> Iterator[AcquireOpenHandlesRecord]:
+        """Return open handles collected by Acquire.
+
+        An Acquire file container contains an open handles csv when the handles module was used. The content of this csv
+        file is returned.
+        """
+        if self.open_handles_file.exists():
+            with self.open_handles_file.open() as fh, gzip.open(fh, "rt") as gz_fh:
+                for row in csv.DictReader(gz_fh):
+                    if name := row.get("name"):
+                        row.update({"name": self.target.fs.path(name)})
+                    yield AcquireOpenHandlesRecord(**row, _target=self.target)

dissect/target/plugins/apps/edr/velociraptor.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+from functools import lru_cache
+from typing import TYPE_CHECKING
+
+from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.helpers.record import DynamicDescriptor, TargetRecordDescriptor
+from dissect.target.plugin import Plugin, export
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from flow.record import Record
+
+    from dissect.target.target import Target
+
+VELOCIRAPTOR_RESULTS = "/$velociraptor_results$"
+ISO_8601_PATTERN = r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?"
+
+
+class VelociraptorRecordBuilder:
+    def __init__(self, artifact_name: str):
+        self._create_event_descriptor = lru_cache(4096)(self._create_event_descriptor)
+        self.record_name = f"velociraptor/{artifact_name}"
+
+    def build(self, object: dict, target: Target) -> TargetRecordDescriptor:
+        """Builds a Velociraptor record."""
+        record_values = {}
+        record_fields = []
+
+        record_values["_target"] = target
+
+        for key, value in object.items():
+            # Reserved by flow.record
+            if key.startswith("_"):
+                continue
+
+            key = key.lower().replace("(", "_").replace(")", "_")
+
+            if re.match(ISO_8601_PATTERN, str(value)):
+                record_type = "datetime"
+            elif isinstance(value, list):
+                record_type = "string[]"
+            elif isinstance(value, int):
+                record_type = "varint"
+            elif key == "hash":
+                record_type = "digest"
+                value = (value.get("MD5"), value.get("SHA1"), value.get("SHA256"))
+            elif isinstance(value, str):
+                record_type = "string"
+            elif isinstance(value, dict):
+                record_type = "record"
+                value = self.build(value, target)
+            else:
+                record_type = "dynamic"
+
+            record_fields.append((record_type, key))
+            record_values[key] = value
+
+        # tuple conversion here is needed for lru_cache
+        desc = self._create_event_descriptor(tuple(record_fields))
+        return desc(**record_values)
+
+    def _create_event_descriptor(self, record_fields: list[tuple[str, str]]) -> TargetRecordDescriptor:
+        return TargetRecordDescriptor(self.record_name, record_fields)
+
+
+class VelociraptorPlugin(Plugin):
+    """Returns records from Velociraptor artifacts."""
+
+    __namespace__ = "velociraptor"
+
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.results_dir = target.fs.path(VELOCIRAPTOR_RESULTS)
+
+    def check_compatible(self) -> None:
+        if not self.results_dir.exists():
+            raise UnsupportedPluginError("No Velociraptor artifacts found")
+
+    @export(record=DynamicDescriptor(["datetime"]))
+    def results(self) -> Iterator[Record]:
+        """Return Rapid7 Velociraptor artifacts.
+
+        References:
+            - https://docs.velociraptor.app/docs/vql/artifacts/
+        """
+        for artifact in self.results_dir.glob("*.json"):
+            # "Windows.KapeFiles.Targets%2FAll\ File\ Metadata.json" becomes "windows_kapefiles_targets"
+            artifact_name = (
+                urllib.parse.unquote(artifact.name.removesuffix(".json")).split("/")[0].lower().replace(".", "_")
+            )
+            record_builder = VelociraptorRecordBuilder(artifact_name)
+
+            for line in artifact.open("rt"):
+                if not (line := line.strip()):
+                    continue
+
+                try:
+                    object = json.loads(line)
+                    yield record_builder.build(object, self.target)
+                except json.decoder.JSONDecodeError:
+                    self.target.log.warning(
+                        "Could not decode Velociraptor JSON log line in file %s: %s",
+                        artifact,
+                        line,
+                    )
+                    continue

tests/loaders/test_velociraptor.py

@@ -24,6 +24,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path:
         f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/$Extend",
         f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/windows/system32",
         f"uploads/{sub_dir}/%5C%5C.%5CC%3A/%2ETEST",
+        "results",
     ]
     root = tmp_path
     mkdirs(root, paths)
@@ -52,16 +53,10 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path:
 
 
 @pytest.mark.parametrize(
-    ("sub_dir", "other_dir"),
-    [
-        ("mft", "auto"),
-        ("ntfs", "auto"),
-        ("ntfs_vss", "auto"),
-        ("lazy_ntfs", "auto"),
-        ("auto", "ntfs"),
-    ],
+    "sub_dir",
+    ["mft", "ntfs", "ntfs_vss", "lazy_ntfs", "auto"],
 )
-def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_path: Path) -> None:
+def test_windows_ntfs(sub_dir: str, target_bare: Target, tmp_path: Path) -> None:
     root = create_root(sub_dir, tmp_path)
 
     assert VelociraptorLoader.detect(root) is True