From 8191b9b48eee045aed0363055b834972f75c3388 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:46:03 +0100 Subject: [PATCH 01/16] improve ApachePlugin log file discovery --- .../target/plugins/apps/webserver/apache.py | 187 ++++++++++++++---- .../target/plugins/apps/webserver/citrix.py | 14 +- tests/plugins/apps/webserver/test_apache.py | 132 +++++++++++-- tests/plugins/apps/webserver/test_citrix.py | 11 +- 4 files changed, 286 insertions(+), 58 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 88b929c7f2..ccf78b8d67 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -1,12 +1,14 @@ +from __future__ import annotations + import itertools import re from datetime import datetime from pathlib import Path -from typing import Iterator, NamedTuple, Optional +from typing import Iterator, NamedTuple -from dissect.target import plugin from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError from dissect.target.helpers.fsutil import open_decompress +from dissect.target.plugin import OperatingSystem, export from dissect.target.plugins.apps.webserver.webserver import ( WebserverAccessLogRecord, WebserverErrorLogRecord, @@ -20,6 +22,31 @@ class LogFormat(NamedTuple): pattern: re.Pattern +# e.g. ServerRoot "/etc/httpd" +RE_CONFIG_ROOT = re.compile( + r""" + [\s#]* # Optionally prefixed by space(s) or pound sign(s). + ServerRoot + \s + "?(?P[^"\s]+)" + $ + """, + re.VERBOSE, +) + +# e.g. Include conf.modules.d/*.conf and IncludeOptional conf.d/*.conf +RE_CONFIG_INCLUDE = re.compile( + r""" + [\s#]* # Optionally prefixed by space(s) or pound sign(s). + (Include|IncludeOptional) # Directive indicating that additional config files are loaded. + \s + ?(?P[^"\s]+) + $ + """, + re.VERBOSE, +) + + # e.g. CustomLog "/custom/log/location/access.log" common RE_CONFIG_CUSTOM_LOG_DIRECTIVE = re.compile( r""" @@ -40,7 +67,7 @@ class LogFormat(NamedTuple): [\s#]* # Optionally prefixed by space(s) or pound sign(s). ErrorLog # Directive indicating that a custom error log location / format is used. \s - "?(?P[^"\s$]+)"? # Location to log to, optionally wrapped in double quotes. + "?(?P[^"\s]+)"? # Location to log to, optionally wrapped in double quotes. $ """, re.VERBOSE, @@ -112,6 +139,8 @@ class LogFormat(NamedTuple): (?P.*) # The actual log message. """ +RE_ENV_VAR_IN_STRING = re.compile(r"\$\{(?P[^\"\s$]+)\}", re.VERBOSE) + LOG_FORMAT_ACCESS_COMMON = LogFormat( "common", re.compile( @@ -187,65 +216,153 @@ class ApachePlugin(WebserverPlugin): "/etc/httpd/conf/httpd.conf", "/etc/httpd.conf", ] + DEFAULT_ENVVAR_PATHS = ["/etc/apache2/envvars", "/etc/sysconfig/httpd", "/etc/rc.conf"] def __init__(self, target: Target): super().__init__(target) - self.access_log_paths, self.error_log_paths = self.get_log_paths() + self.server_root = None + self.access_paths = set() + self.error_paths = set() + self.find_logs() def check_compatible(self) -> None: - if not len(self.access_log_paths) and not len(self.error_log_paths): + if not len(self.access_paths) and not len(self.error_paths): raise UnsupportedPluginError("No Apache directories found") - def get_log_paths(self) -> tuple[list[Path], list[Path]]: - """ - Discover any present Apache log paths on the target system. + if self.target.os in [OperatingSystem.CITRIX, OperatingSystem.FORTIOS]: + raise UnsupportedPluginError("Use other Apache plugin instead") + + def find_logs(self) -> tuple[list[Path], list[Path]]: + """Discover any present Apache log paths on the target system. References: + - https://httpd.apache.org/docs/2.4/logs.html + - https://httpd.apache.org/docs/2.4/mod/mod_log_config.html - https://www.cyberciti.biz/faq/apache-logs/ - https://unix.stackexchange.com/a/269090 """ - access_log_paths = set() - error_log_paths = set() - # Check if any well known default Apache log locations exist for log_dir, log_name in itertools.product(self.DEFAULT_LOG_DIRS, self.ACCESS_LOG_NAMES): - access_log_paths.update(self.target.fs.path(log_dir).glob(f"{log_name}*")) + self.access_paths.update(self.target.fs.path(log_dir).glob(f"*{log_name}*")) for log_dir, log_name in itertools.product(self.DEFAULT_LOG_DIRS, self.ERROR_LOG_NAMES): - error_log_paths.update(self.target.fs.path(log_dir).glob(f"{log_name}*")) + self.error_paths.update(self.target.fs.path(log_dir).glob(f"*{log_name}*")) # Check default Apache configs for CustomLog or ErrorLog directives for config in self.DEFAULT_CONFIG_PATHS: if (path := self.target.fs.path(config)).exists(): - for line in path.open("rt"): - line = line.strip() + self._process_conf_file(path) - if not line or ("CustomLog" not in line and "ErrorLog" not in line): - continue + # Check all .conf files inside the server root + if self.server_root: + for path in self.server_root.rglob("*.conf"): + self._process_conf_file(path) - if "ErrorLog" in line: - set_to_update = error_log_paths - pattern_to_use = RE_CONFIG_ERRORLOG_DIRECTIVE - else: - set_to_update = access_log_paths - pattern_to_use = RE_CONFIG_CUSTOM_LOG_DIRECTIVE + return self.access_paths, self.error_paths - match = pattern_to_use.match(line) - if not match: - self.target.log.warning("Unexpected Apache log configuration: %s (%s)", line, path) + def _process_conf_file(self, path: Path) -> None: + """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude``.""" + + for line in path.open("rt"): + line = line.strip() + + if not line: + continue + + elif "ServerRoot" in line: + match = RE_CONFIG_ROOT.match(line) + if not match: + self.target.log.warning("Unexpected Apache 'ServerRoot' configuration in %s: '%s'", path, line) + continue + directive = match.groupdict() + self.server_root = self.target.fs.path(directive["location"]) + + elif "CustomLog" in line or "ErrorLog" in line: + self._process_conf_line(path, line) + + elif "Include" in line: + match = RE_CONFIG_INCLUDE.match(line) + if not match: + self.target.log.warning("Unexpected Apache 'Include' configuration in %s: '%s'", path, line) + continue + + directive = match.groupdict() + + if "*" in directive["location"]: + root, rest = directive["location"].split("*", 1) + if root.startswith("/"): + root = self.target.fs.path(root) + elif not root.startswith("/") and self.server_root: + root = self.server_root.joinpath(root) + elif not self.server_root: + self.target.log.warning("Unable to resolve relative Include in %s: '%s'", path, line) continue - directive = match.groupdict() - custom_log = self.target.fs.path(directive["location"]) - set_to_update.update(path for path in custom_log.parent.glob(f"{custom_log.name}*")) + for found_conf in root.glob(f"*{rest}"): + self._process_conf_file(found_conf) + + elif ( + self.server_root + and (include_path := self.target.fs.path(self.server_root).joinpath(directive["location"])).exists() + ): + self._process_conf_file(include_path) + + elif (include_path := self.target.fs.path(directive["location"])).exists(): + self._process_conf_file(include_path) + + else: + self.target.log.warning("Unable to resolve Apache Include in %s: '%s'", path, line) + + def _process_conf_line(self, path: Path, line: str) -> None: + """Parse and resolve the given ``CustomLog`` or ``ErrorLog`` directive found in a Apache ``.conf`` file.""" + line = line.strip() + + if "ErrorLog" in line: + pattern = RE_CONFIG_ERRORLOG_DIRECTIVE + set_to_update = self.error_paths + + else: + pattern = RE_CONFIG_CUSTOM_LOG_DIRECTIVE + set_to_update = self.access_paths + + match = pattern.match(line) + if not match: + self.target.log.warning("Unexpected Apache 'ErrorLog' or 'CustomLog' configuration in %s: '%s'", path, line) + return + + directive = match.groupdict() + custom_log = self.target.fs.path(directive["location"]) + match = RE_ENV_VAR_IN_STRING.match(directive["location"]) + + if match: + envvar_directive = match.groupdict() + apache_log_dir = self._read_apache_envvar(envvar_directive["env_var"]) + if apache_log_dir is None: + self.target.log.warning( + "%s does not exist, cannot resolve '%s' in %s", envvar_directive["env_var"], custom_log, path + ) + return + + custom_log = self.target.fs.path( + directive["location"].replace( + r"${" + envvar_directive["env_var"] + "}", apache_log_dir.replace("$SUFFIX", "") + ) + ) + + set_to_update.update(path for path in custom_log.parent.glob(f"*{custom_log.name}*")) - return sorted(access_log_paths), sorted(error_log_paths) + def _read_apache_envvar(self, envvar: str) -> str | None: + for envvar_file in self.DEFAULT_ENVVAR_PATHS: + if (envvars := self.target.fs.path(envvar_file)).exists(): + for line in envvars.read_text().split("\n"): + if f"export {envvar}" in line: + return line.split("=", maxsplit=1)[-1].strip("\"'") - @plugin.export(record=WebserverAccessLogRecord) + @export(record=WebserverAccessLogRecord) def access(self) -> Iterator[WebserverAccessLogRecord]: """Return contents of Apache access log files in unified ``WebserverAccessLogRecord`` format.""" - for line, path in self._iterate_log_lines(self.access_log_paths): + for line, path in self._iterate_log_lines(self.access_paths): try: logformat = self.infer_access_log_format(line) if not logformat: @@ -285,10 +402,10 @@ def access(self) -> Iterator[WebserverAccessLogRecord]: self.target.log.warning("An error occured parsing Apache log file %s: %s", path, str(e)) self.target.log.debug("", exc_info=e) - @plugin.export(record=WebserverErrorLogRecord) + @export(record=WebserverErrorLogRecord) def error(self) -> Iterator[WebserverErrorLogRecord]: """Return contents of Apache error log files in unified ``WebserverErrorLogRecord`` format.""" - for line, path in self._iterate_log_lines(self.error_log_paths): + for line, path in self._iterate_log_lines(self.error_paths): try: match = LOG_FORMAT_ERROR_COMMON.pattern.match(line) if not match: @@ -343,7 +460,7 @@ def _iterate_log_lines(self, paths: list[Path]) -> Iterator[tuple[str, Path]]: self.target.log.warning("Apache log file configured but could not be found (dead symlink?): %s", path) @staticmethod - def infer_access_log_format(line: str) -> Optional[LogFormat]: + def infer_access_log_format(line: str) -> LogFormat | None: """Attempt to infer what standard LogFormat is used. Returns None if no known format can be inferred. Three default log type examples from Apache (note that the ipv4 could also be ipv6) diff --git a/dissect/target/plugins/apps/webserver/citrix.py b/dissect/target/plugins/apps/webserver/citrix.py index 328a0f2a1e..55f27b6570 100644 --- a/dissect/target/plugins/apps/webserver/citrix.py +++ b/dissect/target/plugins/apps/webserver/citrix.py @@ -31,21 +31,21 @@ "combined_resptime_with_citrix_hdrs", re.compile( rf""" - (?P.*?) # Client IP address of the request. + (?P.*?) # Client IP address of the request. \s -> \s - (?P.*?) # Local IP of the Netscaler. + (?P.*?) # Local IP of the Netscaler. \s - (?P.*?) # Remote logname (from identd, if supplied). + (?P.*?) # Remote logname (from identd, if supplied). \s - (?P.*?) # Remote user if the request was authenticated. + (?P.*?) # Remote user if the request was authenticated. \s - {RE_ACCESS_COMMON_PATTERN} # Timestamp, pid, method, uri, protocol, status code, bytes_sent + {RE_ACCESS_COMMON_PATTERN} # Timestamp, pid, method, uri, protocol, status code, bytes_sent \s - {RE_REFERER_USER_AGENT_PATTERN} # Referer, user_agent + {RE_REFERER_USER_AGENT_PATTERN} # Referer, user_agent \s - {RE_RESPONSE_TIME_PATTERN} # Response time + {RE_RESPONSE_TIME_PATTERN} # Response time """, re.VERBOSE, ), diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index 825c2c5399..1587962aeb 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -158,10 +158,10 @@ def test_logrotate(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file("var/log/apache2/access.log.2", data_file) fs_unix.map_file("var/log/apache2/access.log.3", data_file) - access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() + target_unix.add_plugin(ApachePlugin) - assert len(access_log_paths) == 4 - assert len(error_log_paths) == 0 + assert len(target_unix.apache.access_paths) == 4 + assert len(target_unix.apache.error_paths) == 0 def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: @@ -171,10 +171,10 @@ def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file_fh("custom/log/location/access.log.2", BytesIO(b"Foo2")) fs_unix.map_file_fh("custom/log/location/access.log.3", BytesIO(b"Foo3")) - access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() + target_unix.add_plugin(ApachePlugin) - assert len(access_log_paths) == 4 - assert len(error_log_paths) == 0 + assert len(target_unix.apache.access_paths) == 4 + assert len(target_unix.apache.error_paths) == 0 def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) -> None: @@ -183,22 +183,128 @@ def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) CustomLog "/custom/log/location/new.log" common # ErrorLog "/custom/log/location//old_error.log" ErrorLog "/custom/log/location//new_error.log" - """ + fs_unix.map_file_fh("etc/httpd/conf/httpd.conf", BytesIO(textwrap.dedent(config).encode())) fs_unix.map_file_fh("custom/log/location/new.log", BytesIO(b"New")) fs_unix.map_file_fh("custom/log/location/old.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/old_error.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/new_error.log", BytesIO(b"New")) - access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() + target_unix.add_plugin(ApachePlugin) + + assert sorted(list(map(str, target_unix.apache.access_paths))) == [ + "/custom/log/location/new.log", + "/custom/log/location/old.log", + ] + + assert sorted(list(map(str, target_unix.apache.error_paths))) == [ + "/custom/log/location/new_error.log", + "/custom/log/location/old_error.log", + ] + + +def test_config_vhosts_httpd(target_unix: Target, fs_unix: VirtualFilesystem) -> None: + """test if we detect httpd CustomLog and ErrorLog directives using IncludeOptional configuration.""" + + config = """ + ServerRoot "/etc/httpd" + IncludeOptional conf/vhosts/*/*.conf + """ + + # '/etc/httpd/conf/vhosts/*/*.conf' + vhost_config_1 = """ + CustomLog "/custom/log/location/vhost_1.log" common + ErrorLog "/custom/log/location/vhost_1.error" + """ + vhost_config_2 = """ + CustomLog "/custom/log/location/vhost_2.log" combined + ErrorLog "/custom/log/location/vhost_2.error" + """ + fs_unix.map_file_fh("etc/httpd/conf/httpd.conf", BytesIO(textwrap.dedent(config).encode())) + fs_unix.map_file_fh("etc/httpd/conf/vhosts/host1/host1.conf", BytesIO(textwrap.dedent(vhost_config_1).encode())) + fs_unix.map_file_fh("etc/httpd/conf/vhosts/host2/host2.conf", BytesIO(textwrap.dedent(vhost_config_2).encode())) + fs_unix.map_file_fh("custom/log/location/vhost_1.log", BytesIO(b"Log 1")) + fs_unix.map_file_fh("custom/log/location/vhost_2.log", BytesIO(b"Log 2")) + fs_unix.map_file_fh("custom/log/location/vhost_1.error", BytesIO(b"Err 1")) + fs_unix.map_file_fh("custom/log/location/vhost_2.error", BytesIO(b"Err 2")) + target_unix.add_plugin(ApachePlugin) + + access_log_paths, error_log_paths = target_unix.apache.find_logs() + assert sorted(list(map(str, access_log_paths))) == [ + "/custom/log/location/vhost_1.log", + "/custom/log/location/vhost_2.log", + ] + assert sorted(list(map(str, error_log_paths))) == [ + "/custom/log/location/vhost_1.error", + "/custom/log/location/vhost_2.error", + ] + + +def test_config_vhosts_apache2(target_unix: Target, fs_unix: VirtualFilesystem) -> None: + """test if we detect apache2 CustomLog and ErrorLog directives using IncludeOptional configuration.""" + + config = r""" + ServerRoot "/etc/apache2" + ErrorLog ${APACHE_LOG_DIR}/error.log + Include example.conf + IncludeOptional conf-enabled/*.conf + IncludeOptional sites-enabled/*.conf + """ + fs_unix.map_file_fh("/etc/apache2/apache2.conf", BytesIO(textwrap.dedent(config).encode())) + fs_unix.map_file_fh("/path/to/apache/logs/error.log", BytesIO(b"")) + + envvars = r""" + export APACHE_LOG_DIR="/path/to/apache/logs" + """ + fs_unix.map_file_fh("/etc/apache2/envvars", BytesIO(textwrap.dedent(envvars).encode())) - # Log paths are returned in alphabetical order - assert str(access_log_paths[0]) == "/custom/log/location/new.log" - assert str(access_log_paths[1]) == "/custom/log/location/old.log" + enabled_conf = r""" + CustomLog ${APACHE_LOG_DIR}/example_access.log custom + """ + fs_unix.map_file_fh("/etc/apache2/conf-enabled/example.conf", BytesIO(textwrap.dedent(enabled_conf).encode())) + fs_unix.map_file_fh("/path/to/apache/logs/example_access.log.1.gz", BytesIO(b"")) + + site_conf = """ + + ServerName example.com + DocumentRoot /var/www/html + ErrorLog /path/to/virtualhost/log/error.log + CustomLog /path/to/virtualhost/log/access.log custom + + """ + fs_unix.map_file_fh("/etc/apache2/sites-enabled/example.conf", BytesIO(textwrap.dedent(site_conf).encode())) + fs_unix.map_file_fh("/path/to/virtualhost/log/error.log.1", BytesIO(b"")) + fs_unix.map_file_fh("/path/to/virtualhost/log/access.log.1", BytesIO(b"")) + + disabled_conf = """ + CustomLog /path/to/disabled/access.log custom + """ + fs_unix.map_file_fh( + "/etc/apache2/sites-available/disabled-site.conf", BytesIO(textwrap.dedent(disabled_conf).encode()) + ) + fs_unix.map_file_fh("/path/to/disabled/access.log.2", BytesIO(b"")) + + fs_unix.map_file_fh("/var/log/apache2/some-other-vhost-old-log.access.log", BytesIO(b"")) + + fs_unix.map_file_fh("/var/log/apache2/error.log", BytesIO(b"")) + fs_unix.map_file_fh("/var/log/apache2/access.log", BytesIO(b"")) + + target_unix.add_plugin(ApachePlugin) - assert str(error_log_paths[0]) == "/custom/log/location/new_error.log" - assert str(error_log_paths[1]) == "/custom/log/location/old_error.log" + assert sorted(list(map(str, target_unix.apache.access_paths))) == [ + "/path/to/apache/logs/example_access.log.1.gz", + "/path/to/disabled/access.log.2", + "/path/to/virtualhost/log/access.log.1", + "/var/log/apache2/access.log", + "/var/log/apache2/some-other-vhost-old-log.access.log", + ] + + assert sorted(list(map(str, target_unix.apache.error_paths))) == [ + "/path/to/apache/logs/error.log", + "/path/to/virtualhost/log/error.log.1", + "/var/log/apache2/error.log", + ] def test_error_txt(target_unix: Target, fs_unix: VirtualFilesystem) -> None: diff --git a/tests/plugins/apps/webserver/test_citrix.py b/tests/plugins/apps/webserver/test_citrix.py index 32f8567421..5b16e525b9 100644 --- a/tests/plugins/apps/webserver/test_citrix.py +++ b/tests/plugins/apps/webserver/test_citrix.py @@ -1,6 +1,9 @@ from datetime import datetime, timedelta, timezone from io import BytesIO +import pytest + +from dissect.target.exceptions import UnsupportedPluginError from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.apps.webserver.apache import ApachePlugin from dissect.target.plugins.apps.webserver.citrix import ( @@ -75,16 +78,18 @@ def test_error_logs(target_citrix: Target, fs_bsd: VirtualFilesystem) -> None: fs_bsd.map_file("var/log/httperror-vpn.log", BytesIO(b"Foo")) fs_bsd.map_file("var/log/httperror.log", BytesIO(b"Bar")) - access_log_paths, error_log_paths = CitrixWebserverPlugin(target_citrix).get_log_paths() + target_citrix.add_plugin(CitrixWebserverPlugin) - assert len(error_log_paths) == 2 + assert len(target_citrix.citrix.error_paths) == 2 def test_access_logs_webserver_namespace(target_citrix: Target, fs_bsd: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/citrix/httpaccess.log") fs_bsd.map_file("var/log/httpaccess.log", data_file) - target_citrix.add_plugin(ApachePlugin, check_compatible=False) + with pytest.raises(UnsupportedPluginError): + target_citrix.add_plugin(ApachePlugin) + target_citrix.add_plugin(CitrixWebserverPlugin) target_citrix.add_plugin(WebserverPlugin) From f7644aa777e8d4f5decbf8e6c01e7484c4154b15 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 25 Feb 2025 14:47:12 +0100 Subject: [PATCH 02/16] prevent recursion --- dissect/target/plugins/apps/webserver/apache.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index ccf78b8d67..ac7afd20be 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -242,6 +242,8 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: - https://unix.stackexchange.com/a/269090 """ + self.seen_confs = set() + # Check if any well known default Apache log locations exist for log_dir, log_name in itertools.product(self.DEFAULT_LOG_DIRS, self.ACCESS_LOG_NAMES): self.access_paths.update(self.target.fs.path(log_dir).glob(f"*{log_name}*")) @@ -262,7 +264,12 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: return self.access_paths, self.error_paths def _process_conf_file(self, path: Path) -> None: - """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude``.""" + """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives.""" + + if path in self.seen_confs: + self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) + return + self.seen_confs.add(path) for line in path.open("rt"): line = line.strip() From 2e82d545a147e573f34deba79fe514a6632df273 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:48:00 +0100 Subject: [PATCH 03/16] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- .../target/plugins/apps/webserver/apache.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index ac7afd20be..eac3d10f6c 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -280,7 +280,7 @@ def _process_conf_file(self, path: Path) -> None: elif "ServerRoot" in line: match = RE_CONFIG_ROOT.match(line) if not match: - self.target.log.warning("Unexpected Apache 'ServerRoot' configuration in %s: '%s'", path, line) + self.target.log.warning("Unexpected Apache 'ServerRoot' configuration in %s: %r", path, line) continue directive = match.groupdict() self.server_root = self.target.fs.path(directive["location"]) @@ -291,7 +291,7 @@ def _process_conf_file(self, path: Path) -> None: elif "Include" in line: match = RE_CONFIG_INCLUDE.match(line) if not match: - self.target.log.warning("Unexpected Apache 'Include' configuration in %s: '%s'", path, line) + self.target.log.warning("Unexpected Apache 'Include' configuration in %s: %r", path, line) continue directive = match.groupdict() @@ -303,7 +303,7 @@ def _process_conf_file(self, path: Path) -> None: elif not root.startswith("/") and self.server_root: root = self.server_root.joinpath(root) elif not self.server_root: - self.target.log.warning("Unable to resolve relative Include in %s: '%s'", path, line) + self.target.log.warning("Unable to resolve relative Include in %s: %r", path, line) continue for found_conf in root.glob(f"*{rest}"): @@ -311,7 +311,7 @@ def _process_conf_file(self, path: Path) -> None: elif ( self.server_root - and (include_path := self.target.fs.path(self.server_root).joinpath(directive["location"])).exists() + and (include_path := self.server_root.joinpath(directive["location"])).exists() ): self._process_conf_file(include_path) @@ -319,7 +319,7 @@ def _process_conf_file(self, path: Path) -> None: self._process_conf_file(include_path) else: - self.target.log.warning("Unable to resolve Apache Include in %s: '%s'", path, line) + self.target.log.warning("Unable to resolve Apache Include in %s: %r", path, line) def _process_conf_line(self, path: Path, line: str) -> None: """Parse and resolve the given ``CustomLog`` or ``ErrorLog`` directive found in a Apache ``.conf`` file.""" @@ -335,14 +335,12 @@ def _process_conf_line(self, path: Path, line: str) -> None: match = pattern.match(line) if not match: - self.target.log.warning("Unexpected Apache 'ErrorLog' or 'CustomLog' configuration in %s: '%s'", path, line) + self.target.log.warning("Unexpected Apache 'ErrorLog' or 'CustomLog' configuration in %s: %r", path, line) return directive = match.groupdict() custom_log = self.target.fs.path(directive["location"]) - match = RE_ENV_VAR_IN_STRING.match(directive["location"]) - - if match: + if match := RE_ENV_VAR_IN_STRING.match(directive["location"]): envvar_directive = match.groupdict() apache_log_dir = self._read_apache_envvar(envvar_directive["env_var"]) if apache_log_dir is None: @@ -353,7 +351,7 @@ def _process_conf_line(self, path: Path, line: str) -> None: custom_log = self.target.fs.path( directive["location"].replace( - r"${" + envvar_directive["env_var"] + "}", apache_log_dir.replace("$SUFFIX", "") + f"${{{envvar_directive['env_var']}}}", apache_log_dir.replace("$SUFFIX", "") ) ) From 49dfa1ede6491535f80ded988ea0805fcd6f3fa3 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:51:08 +0100 Subject: [PATCH 04/16] implement review feedback --- .../target/plugins/apps/webserver/apache.py | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index eac3d10f6c..dcc20775ef 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -229,8 +229,8 @@ def check_compatible(self) -> None: if not len(self.access_paths) and not len(self.error_paths): raise UnsupportedPluginError("No Apache directories found") - if self.target.os in [OperatingSystem.CITRIX, OperatingSystem.FORTIOS]: - raise UnsupportedPluginError("Use other Apache plugin instead") + if self.target.os == OperatingSystem.CITRIX: + raise UnsupportedPluginError(f"Use {self.target.os} Apache plugin instead") def find_logs(self) -> tuple[list[Path], list[Path]]: """Discover any present Apache log paths on the target system. @@ -242,8 +242,6 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: - https://unix.stackexchange.com/a/269090 """ - self.seen_confs = set() - # Check if any well known default Apache log locations exist for log_dir, log_name in itertools.product(self.DEFAULT_LOG_DIRS, self.ACCESS_LOG_NAMES): self.access_paths.update(self.target.fs.path(log_dir).glob(f"*{log_name}*")) @@ -263,13 +261,14 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: return self.access_paths, self.error_paths - def _process_conf_file(self, path: Path) -> None: + def _process_conf_file(self, path: Path, seen: set[Path] | None) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives.""" + seen = seen or set() - if path in self.seen_confs: + if path in seen: self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) return - self.seen_confs.add(path) + seen.add(path) for line in path.open("rt"): line = line.strip() @@ -295,9 +294,10 @@ def _process_conf_file(self, path: Path) -> None: continue directive = match.groupdict() + location = directive["location"] - if "*" in directive["location"]: - root, rest = directive["location"].split("*", 1) + if "*" in location: + root, rest = location.split("*", 1) if root.startswith("/"): root = self.target.fs.path(root) elif not root.startswith("/") and self.server_root: @@ -307,16 +307,13 @@ def _process_conf_file(self, path: Path) -> None: continue for found_conf in root.glob(f"*{rest}"): - self._process_conf_file(found_conf) + self._process_conf_file(found_conf, seen) - elif ( - self.server_root - and (include_path := self.server_root.joinpath(directive["location"])).exists() - ): - self._process_conf_file(include_path) + elif self.server_root and (include_path := self.server_root.joinpath(location)).exists(): + self._process_conf_file(include_path, seen) - elif (include_path := self.target.fs.path(directive["location"])).exists(): - self._process_conf_file(include_path) + elif (include_path := self.target.fs.path(location)).exists(): + self._process_conf_file(include_path, seen) else: self.target.log.warning("Unable to resolve Apache Include in %s: %r", path, line) @@ -339,20 +336,19 @@ def _process_conf_line(self, path: Path, line: str) -> None: return directive = match.groupdict() - custom_log = self.target.fs.path(directive["location"]) - if match := RE_ENV_VAR_IN_STRING.match(directive["location"]): + location = directive["location"] + custom_log = self.target.fs.path(location) + + if match := RE_ENV_VAR_IN_STRING.match(location): envvar_directive = match.groupdict() - apache_log_dir = self._read_apache_envvar(envvar_directive["env_var"]) + env_var = envvar_directive["env_var"] + apache_log_dir = self._read_apache_envvar(env_var) if apache_log_dir is None: - self.target.log.warning( - "%s does not exist, cannot resolve '%s' in %s", envvar_directive["env_var"], custom_log, path - ) + self.target.log.warning("%s does not exist, cannot resolve '%s' in %s", env_var, custom_log, path) return custom_log = self.target.fs.path( - directive["location"].replace( - f"${{{envvar_directive['env_var']}}}", apache_log_dir.replace("$SUFFIX", "") - ) + directive["location"].replace(f"${{{env_var}}}", apache_log_dir.replace("$SUFFIX", "")) ) set_to_update.update(path for path in custom_log.parent.glob(f"*{custom_log.name}*")) From 7e6f897234ab70d93508b0302554f623f5a9f191 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:55:35 +0100 Subject: [PATCH 05/16] fix linter --- dissect/target/plugins/apps/webserver/apache.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index dcc20775ef..82a7b07caf 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -261,13 +261,15 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: return self.access_paths, self.error_paths - def _process_conf_file(self, path: Path, seen: set[Path] | None) -> None: - """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives.""" - seen = seen or set() + def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None: + """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` + and ``OptionalInclude`` directives. + """ if path in seen: - self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) + self.target.log.info("Detected recursion in Apache configuration, file already parsed: %s", path) return + seen.add(path) for line in path.open("rt"): @@ -279,7 +281,7 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None) -> None: elif "ServerRoot" in line: match = RE_CONFIG_ROOT.match(line) if not match: - self.target.log.warning("Unexpected Apache 'ServerRoot' configuration in %s: %r", path, line) + self.target.log.warning("Unable to parse Apache 'ServerRoot' configuration in %s: %r", path, line) continue directive = match.groupdict() self.server_root = self.target.fs.path(directive["location"]) @@ -290,7 +292,7 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None) -> None: elif "Include" in line: match = RE_CONFIG_INCLUDE.match(line) if not match: - self.target.log.warning("Unexpected Apache 'Include' configuration in %s: %r", path, line) + self.target.log.warning("Unable to parse Apache 'Include' configuration in %s: %r", path, line) continue directive = match.groupdict() From a723363ac93deeb479e5f98cd02e2f75e168ab69 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 5 Mar 2025 14:57:25 +0100 Subject: [PATCH 06/16] prevent additional recursion and cache env vars --- .../target/plugins/apps/webserver/apache.py | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 82a7b07caf..e62e8022ec 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -3,6 +3,7 @@ import itertools import re from datetime import datetime +from functools import cached_property from pathlib import Path from typing import Iterator, NamedTuple @@ -226,14 +227,15 @@ def __init__(self, target: Target): self.find_logs() def check_compatible(self) -> None: - if not len(self.access_paths) and not len(self.error_paths): - raise UnsupportedPluginError("No Apache directories found") + if not self.access_paths and not self.error_paths: + raise UnsupportedPluginError("No Apache log files found") if self.target.os == OperatingSystem.CITRIX: raise UnsupportedPluginError(f"Use {self.target.os} Apache plugin instead") - def find_logs(self) -> tuple[list[Path], list[Path]]: + def find_logs(self) -> None: """Discover any present Apache log paths on the target system. + Populates ``self.access_paths`` and ``self.error_paths``. References: - https://httpd.apache.org/docs/2.4/logs.html @@ -249,25 +251,28 @@ def find_logs(self) -> tuple[list[Path], list[Path]]: for log_dir, log_name in itertools.product(self.DEFAULT_LOG_DIRS, self.ERROR_LOG_NAMES): self.error_paths.update(self.target.fs.path(log_dir).glob(f"*{log_name}*")) + seen = set() + # Check default Apache configs for CustomLog or ErrorLog directives for config in self.DEFAULT_CONFIG_PATHS: - if (path := self.target.fs.path(config)).exists(): + if (path := self.target.fs.path(config)).exists() and path not in seen: + seen.add(path) self._process_conf_file(path) # Check all .conf files inside the server root if self.server_root: for path in self.server_root.rglob("*.conf"): - self._process_conf_file(path) - - return self.access_paths, self.error_paths + if path not in seen: + seen.add(path) + self._process_conf_file(path) def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` - and ``OptionalInclude`` directives. + and ``OptionalInclude`` directives. Populates ``self.access_paths`` and ``self.error_paths``. """ if path in seen: - self.target.log.info("Detected recursion in Apache configuration, file already parsed: %s", path) + self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) return seen.add(path) @@ -283,8 +288,8 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None if not match: self.target.log.warning("Unable to parse Apache 'ServerRoot' configuration in %s: %r", path, line) continue - directive = match.groupdict() - self.server_root = self.target.fs.path(directive["location"]) + location = match.groupdict().get("location") + self.server_root = self.target.fs.path(location) elif "CustomLog" in line or "ErrorLog" in line: self._process_conf_line(path, line) @@ -295,8 +300,7 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None self.target.log.warning("Unable to parse Apache 'Include' configuration in %s: %r", path, line) continue - directive = match.groupdict() - location = directive["location"] + location = match.groupdict().get("location") if "*" in location: root, rest = location.split("*", 1) @@ -337,30 +341,31 @@ def _process_conf_line(self, path: Path, line: str) -> None: self.target.log.warning("Unexpected Apache 'ErrorLog' or 'CustomLog' configuration in %s: %r", path, line) return - directive = match.groupdict() - location = directive["location"] + location = match.groupdict().get("location") custom_log = self.target.fs.path(location) - if match := RE_ENV_VAR_IN_STRING.match(location): - envvar_directive = match.groupdict() + if env_var_match := RE_ENV_VAR_IN_STRING.match(location): + envvar_directive = env_var_match.groupdict() env_var = envvar_directive["env_var"] - apache_log_dir = self._read_apache_envvar(env_var) + apache_log_dir = self.env_vars.get(env_var) if apache_log_dir is None: self.target.log.warning("%s does not exist, cannot resolve '%s' in %s", env_var, custom_log, path) return - custom_log = self.target.fs.path( - directive["location"].replace(f"${{{env_var}}}", apache_log_dir.replace("$SUFFIX", "")) - ) + custom_log = self.target.fs.path(location.replace(f"${{{env_var}}}", apache_log_dir.replace("$SUFFIX", ""))) set_to_update.update(path for path in custom_log.parent.glob(f"*{custom_log.name}*")) - def _read_apache_envvar(self, envvar: str) -> str | None: + @cached_property + def env_vars(self) -> dict: + variables = {} for envvar_file in self.DEFAULT_ENVVAR_PATHS: - if (envvars := self.target.fs.path(envvar_file)).exists(): - for line in envvars.read_text().split("\n"): - if f"export {envvar}" in line: - return line.split("=", maxsplit=1)[-1].strip("\"'") + if (file := self.target.fs.path(envvar_file)).exists(): + for line in file.read_text().split("\n"): + key, _, value = line.partition("=") + if key: + variables[key.replace("export ", "")] = value.strip("\"'") + return variables @export(record=WebserverAccessLogRecord) def access(self) -> Iterator[WebserverAccessLogRecord]: From 8e127b87033b20fa3ccb624e344b07a174b39b25 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:28:00 +0100 Subject: [PATCH 07/16] use factory instead (tests still fail) --- tests/conftest.py | 22 +++++++++++ tests/plugins/apps/webserver/test_apache.py | 43 ++++++++++++++------- tests/tools/test_diff.py | 27 +------------ 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ce95a1a9d8..0fab347419 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -491,3 +491,25 @@ def target_linux_docker(tmp_path: pathlib.Path, fs_docker: TarFilesystem) -> Ite mock_target.fs.mount("/", fs_docker) mock_target.apply() yield mock_target + + +class TargetUnixFactory: + def __init__(self, tmp_path: pathlib.Path): + self.tmp_path = tmp_path + + def new(self, hostname: str = "hostname") -> tuple[Target, VirtualFilesystem]: + """Initialize a virtual unix target.""" + fs = VirtualFilesystem() + + fs.makedirs("var") + fs.makedirs("etc") + fs.map_file_fh("/etc/hostname", BytesIO(hostname.encode())) + + return make_os_target(self.tmp_path, UnixPlugin, root_fs=fs), fs + + +@pytest.fixture +def target_unix_factory(tmp_path: pathlib.Path) -> TargetUnixFactory: + """This fixture returns a class that can instantiate a virtual unix targets from a blueprint. This can then be used + to create a fixture for the source target and the desination target, without them 'bleeding' into each other.""" + return TargetUnixFactory(tmp_path) diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index 1587962aeb..a66d132b75 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -11,6 +11,7 @@ ) from dissect.target.target import Target from tests._utils import absolute_path +from tests.conftest import TargetUnixFactory def test_infer_access_log_format_combined() -> None: @@ -164,20 +165,30 @@ def test_logrotate(target_unix: Target, fs_unix: VirtualFilesystem) -> None: assert len(target_unix.apache.error_paths) == 0 -def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: - fs_unix.map_file_fh("etc/apache2/apache2.conf", BytesIO(b'CustomLog "/custom/log/location/access.log" common')) - fs_unix.map_file_fh("custom/log/location/access.log", BytesIO(b"Foo")) - fs_unix.map_file_fh("custom/log/location/access.log.1", BytesIO(b"Foo1")) - fs_unix.map_file_fh("custom/log/location/access.log.2", BytesIO(b"Foo2")) - fs_unix.map_file_fh("custom/log/location/access.log.3", BytesIO(b"Foo3")) +def test_custom_config(target_unix_factory: TargetUnixFactory) -> None: + target_unix, fs_unix = target_unix_factory.new() + fs_unix.map_file_fh( + "/etc/apache2/apache2.conf", BytesIO(b'CustomLog "/very/custom/log/location/access.log" common') + ) + fs_unix.map_file_fh("/very/custom/log/location/access.log", BytesIO(b"Foo")) + fs_unix.map_file_fh("/very/custom/log/location/access.log.1", BytesIO(b"Foo1")) + fs_unix.map_file_fh("/very/custom/log/location/access.log.2", BytesIO(b"Foo2")) + fs_unix.map_file_fh("/very/custom/log/location/access.log.3", BytesIO(b"Foo3")) target_unix.add_plugin(ApachePlugin) - assert len(target_unix.apache.access_paths) == 4 + assert sorted(list(map(str, target_unix.apache.access_paths))) == [ + "/very/custom/log/location/access.log", + "/very/custom/log/location/access.log.1", + "/very/custom/log/location/access.log.2", + "/very/custom/log/location/access.log.3", + ] assert len(target_unix.apache.error_paths) == 0 -def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) -> None: +def test_config_commented_logs(target_unix_factory: TargetUnixFactory) -> None: + target_unix, fs_unix = target_unix_factory.new() + config = """ # CustomLog "/custom/log/location/old.log" common CustomLog "/custom/log/location/new.log" common @@ -204,9 +215,11 @@ def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) ] -def test_config_vhosts_httpd(target_unix: Target, fs_unix: VirtualFilesystem) -> None: +def test_config_vhosts_httpd(target_unix_factory: TargetUnixFactory) -> None: """test if we detect httpd CustomLog and ErrorLog directives using IncludeOptional configuration.""" + target_unix, fs_unix = target_unix_factory.new() + config = """ ServerRoot "/etc/httpd" IncludeOptional conf/vhosts/*/*.conf @@ -230,20 +243,21 @@ def test_config_vhosts_httpd(target_unix: Target, fs_unix: VirtualFilesystem) -> fs_unix.map_file_fh("custom/log/location/vhost_2.error", BytesIO(b"Err 2")) target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.find_logs() - assert sorted(list(map(str, access_log_paths))) == [ + assert sorted(list(map(str, target_unix.apache.access_paths))) == [ "/custom/log/location/vhost_1.log", "/custom/log/location/vhost_2.log", ] - assert sorted(list(map(str, error_log_paths))) == [ + assert sorted(list(map(str, target_unix.apache.error_paths))) == [ "/custom/log/location/vhost_1.error", "/custom/log/location/vhost_2.error", ] -def test_config_vhosts_apache2(target_unix: Target, fs_unix: VirtualFilesystem) -> None: +def test_config_vhosts_apache2(target_unix_factory: TargetUnixFactory) -> None: """test if we detect apache2 CustomLog and ErrorLog directives using IncludeOptional configuration.""" + target_unix, fs_unix = target_unix_factory.new() + config = r""" ServerRoot "/etc/apache2" ErrorLog ${APACHE_LOG_DIR}/error.log @@ -286,9 +300,8 @@ def test_config_vhosts_apache2(target_unix: Target, fs_unix: VirtualFilesystem) fs_unix.map_file_fh("/path/to/disabled/access.log.2", BytesIO(b"")) fs_unix.map_file_fh("/var/log/apache2/some-other-vhost-old-log.access.log", BytesIO(b"")) - - fs_unix.map_file_fh("/var/log/apache2/error.log", BytesIO(b"")) fs_unix.map_file_fh("/var/log/apache2/access.log", BytesIO(b"")) + fs_unix.map_file_fh("/var/log/apache2/error.log", BytesIO(b"")) target_unix.add_plugin(ApachePlugin) diff --git a/tests/tools/test_diff.py b/tests/tools/test_diff.py index 329014271f..9169c01558 100644 --- a/tests/tools/test_diff.py +++ b/tests/tools/test_diff.py @@ -2,14 +2,11 @@ import textwrap from io import BytesIO, StringIO -from pathlib import Path from typing import Iterator import pytest -from dissect.target.filesystem import VirtualFilesystem from dissect.target.helpers.fsutil import stat_result -from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.target import Target from dissect.target.tools import fsutils from dissect.target.tools.diff import ( @@ -21,7 +18,7 @@ ) from dissect.target.tools.diff import main as target_diff from tests._utils import absolute_path -from tests.conftest import make_os_target +from tests.conftest import TargetUnixFactory PASSWD_CONTENTS = """ root:x:0:0:root:/root:/bin/bash @@ -29,28 +26,6 @@ """ -class TargetUnixFactory: - def __init__(self, tmp_path: Path): - self.tmp_path = tmp_path - - def new(self, hostname: str) -> tuple[Target, VirtualFilesystem]: - """Initialize a virtual unix target.""" - fs = VirtualFilesystem() - - fs.makedirs("var") - fs.makedirs("etc") - fs.map_file_fh("/etc/hostname", BytesIO(hostname.encode())) - - return make_os_target(self.tmp_path, UnixPlugin, root_fs=fs), fs - - -@pytest.fixture -def target_unix_factory(tmp_path: Path) -> TargetUnixFactory: - """This fixture returns a class that can instantiate a virtual unix targets from a blueprint. This can then be used - to create a fixture for the source target and the desination target, without them 'bleeding' into each other.""" - return TargetUnixFactory(tmp_path) - - @pytest.fixture def src_target(target_unix_factory: TargetUnixFactory) -> Iterator[Target]: target, fs_unix = target_unix_factory.new("src_target") From d86bd333996f114283fc5928faa42d7c54cec7cf Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:54:54 +0100 Subject: [PATCH 08/16] explicitly setting seen to None is a workaround for the caching bug --- dissect/target/plugins/apps/webserver/apache.py | 10 ++++++++-- tests/plugins/apps/webserver/test_apache.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index e62e8022ec..ef7dd88d6b 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -256,21 +256,27 @@ def find_logs(self) -> None: # Check default Apache configs for CustomLog or ErrorLog directives for config in self.DEFAULT_CONFIG_PATHS: if (path := self.target.fs.path(config)).exists() and path not in seen: + self._process_conf_file(path, seen=None) seen.add(path) - self._process_conf_file(path) # Check all .conf files inside the server root if self.server_root: for path in self.server_root.rglob("*.conf"): if path not in seen: + self._process_conf_file(path, seen=None) seen.add(path) - self._process_conf_file(path) def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives. Populates ``self.access_paths`` and ``self.error_paths``. """ + self.target.log.debug("Processing conf file: %s", path) + self.target.log.debug("Current seen conf files: %s", seen) + + if not seen: + seen = set() + if path in seen: self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) return diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index a66d132b75..f81daae060 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -10,6 +10,7 @@ ApachePlugin, ) from dissect.target.target import Target + from tests._utils import absolute_path from tests.conftest import TargetUnixFactory @@ -241,6 +242,7 @@ def test_config_vhosts_httpd(target_unix_factory: TargetUnixFactory) -> None: fs_unix.map_file_fh("custom/log/location/vhost_2.log", BytesIO(b"Log 2")) fs_unix.map_file_fh("custom/log/location/vhost_1.error", BytesIO(b"Err 1")) fs_unix.map_file_fh("custom/log/location/vhost_2.error", BytesIO(b"Err 2")) + target_unix.add_plugin(ApachePlugin) assert sorted(list(map(str, target_unix.apache.access_paths))) == [ From a3531a97e8da3eed251678f0c4f3b546db01db92 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:11:20 +0100 Subject: [PATCH 09/16] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/target/plugins/apps/webserver/apache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index ef7dd88d6b..8783a92ed6 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -266,7 +266,7 @@ def find_logs(self) -> None: self._process_conf_file(path, seen=None) seen.add(path) - def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None: + def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives. Populates ``self.access_paths`` and ``self.error_paths``. """ @@ -274,8 +274,7 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = set()) -> None self.target.log.debug("Processing conf file: %s", path) self.target.log.debug("Current seen conf files: %s", seen) - if not seen: - seen = set() + seen = seen or set() if path in seen: self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path) From 85387adb1fca90be91f1bdd943242f52641160ef Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:17:58 +0100 Subject: [PATCH 10/16] implement review feedback --- .../target/plugins/apps/webserver/apache.py | 8 ++------ tests/plugins/apps/webserver/test_apache.py | 18 ++++-------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 8783a92ed6..0bfa50ae46 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -256,24 +256,20 @@ def find_logs(self) -> None: # Check default Apache configs for CustomLog or ErrorLog directives for config in self.DEFAULT_CONFIG_PATHS: if (path := self.target.fs.path(config)).exists() and path not in seen: - self._process_conf_file(path, seen=None) + self._process_conf_file(path) seen.add(path) # Check all .conf files inside the server root if self.server_root: for path in self.server_root.rglob("*.conf"): if path not in seen: - self._process_conf_file(path, seen=None) + self._process_conf_file(path) seen.add(path) def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives. Populates ``self.access_paths`` and ``self.error_paths``. """ - - self.target.log.debug("Processing conf file: %s", path) - self.target.log.debug("Current seen conf files: %s", seen) - seen = seen or set() if path in seen: diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index f81daae060..4c6a8eb244 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -12,7 +12,6 @@ from dissect.target.target import Target from tests._utils import absolute_path -from tests.conftest import TargetUnixFactory def test_infer_access_log_format_combined() -> None: @@ -166,8 +165,7 @@ def test_logrotate(target_unix: Target, fs_unix: VirtualFilesystem) -> None: assert len(target_unix.apache.error_paths) == 0 -def test_custom_config(target_unix_factory: TargetUnixFactory) -> None: - target_unix, fs_unix = target_unix_factory.new() +def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file_fh( "/etc/apache2/apache2.conf", BytesIO(b'CustomLog "/very/custom/log/location/access.log" common') ) @@ -187,9 +185,7 @@ def test_custom_config(target_unix_factory: TargetUnixFactory) -> None: assert len(target_unix.apache.error_paths) == 0 -def test_config_commented_logs(target_unix_factory: TargetUnixFactory) -> None: - target_unix, fs_unix = target_unix_factory.new() - +def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ # CustomLog "/custom/log/location/old.log" common CustomLog "/custom/log/location/new.log" common @@ -216,11 +212,8 @@ def test_config_commented_logs(target_unix_factory: TargetUnixFactory) -> None: ] -def test_config_vhosts_httpd(target_unix_factory: TargetUnixFactory) -> None: +def test_config_vhosts_httpd(target_unix: Target, fs_unix: VirtualFilesystem) -> None: """test if we detect httpd CustomLog and ErrorLog directives using IncludeOptional configuration.""" - - target_unix, fs_unix = target_unix_factory.new() - config = """ ServerRoot "/etc/httpd" IncludeOptional conf/vhosts/*/*.conf @@ -255,11 +248,8 @@ def test_config_vhosts_httpd(target_unix_factory: TargetUnixFactory) -> None: ] -def test_config_vhosts_apache2(target_unix_factory: TargetUnixFactory) -> None: +def test_config_vhosts_apache2(target_unix: Target, fs_unix: VirtualFilesystem) -> None: """test if we detect apache2 CustomLog and ErrorLog directives using IncludeOptional configuration.""" - - target_unix, fs_unix = target_unix_factory.new() - config = r""" ServerRoot "/etc/apache2" ErrorLog ${APACHE_LOG_DIR}/error.log From 01c55465c422f5cbddd035818fc16fa26044cca5 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:20:49 +0100 Subject: [PATCH 11/16] fix linter --- tests/plugins/apps/webserver/test_apache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index 4c6a8eb244..4a4a2ea214 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -10,7 +10,6 @@ ApachePlugin, ) from dissect.target.target import Target - from tests._utils import absolute_path From 20ace1faec1b043f23036d20f26ca680bb511417 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:49:52 +0100 Subject: [PATCH 12/16] implement review feedback --- dissect/target/plugins/apps/webserver/apache.py | 2 +- tests/plugins/apps/webserver/test_citrix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 0bfa50ae46..8ab51da0f6 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -231,7 +231,7 @@ def check_compatible(self) -> None: raise UnsupportedPluginError("No Apache log files found") if self.target.os == OperatingSystem.CITRIX: - raise UnsupportedPluginError(f"Use {self.target.os} Apache plugin instead") + raise UnsupportedPluginError("Use the 'apps.webserver.citrix' apache plugin instead") def find_logs(self) -> None: """Discover any present Apache log paths on the target system. diff --git a/tests/plugins/apps/webserver/test_citrix.py b/tests/plugins/apps/webserver/test_citrix.py index 5b16e525b9..12bf792203 100644 --- a/tests/plugins/apps/webserver/test_citrix.py +++ b/tests/plugins/apps/webserver/test_citrix.py @@ -87,7 +87,7 @@ def test_access_logs_webserver_namespace(target_citrix: Target, fs_bsd: VirtualF data_file = absolute_path("_data/plugins/apps/webserver/citrix/httpaccess.log") fs_bsd.map_file("var/log/httpaccess.log", data_file) - with pytest.raises(UnsupportedPluginError): + with pytest.raises(UnsupportedPluginError, match="Use the 'apps.webserver.citrix' apache plugin instead"): target_citrix.add_plugin(ApachePlugin) target_citrix.add_plugin(CitrixWebserverPlugin) From 276c5a305641ce74550c9aa6e92954de307ecf50 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:46:27 +0100 Subject: [PATCH 13/16] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- .../target/plugins/apps/webserver/apache.py | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 8ab51da0f6..81118ddd45 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -279,14 +279,11 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: seen.add(path) for line in path.open("rt"): - line = line.strip() - - if not line: + if not (line := line.strip()): continue - elif "ServerRoot" in line: - match = RE_CONFIG_ROOT.match(line) - if not match: + if "ServerRoot" in line: + if not (match := RE_CONFIG_ROOT.match(line)): self.target.log.warning("Unable to parse Apache 'ServerRoot' configuration in %s: %r", path, line) continue location = match.groupdict().get("location") @@ -296,8 +293,7 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: self._process_conf_line(path, line) elif "Include" in line: - match = RE_CONFIG_INCLUDE.match(line) - if not match: + if not (match := RE_CONFIG_INCLUDE.match(line)): self.target.log.warning("Unable to parse Apache 'Include' configuration in %s: %r", path, line) continue @@ -327,18 +323,14 @@ def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: def _process_conf_line(self, path: Path, line: str) -> None: """Parse and resolve the given ``CustomLog`` or ``ErrorLog`` directive found in a Apache ``.conf`` file.""" - line = line.strip() - if "ErrorLog" in line: pattern = RE_CONFIG_ERRORLOG_DIRECTIVE set_to_update = self.error_paths - else: pattern = RE_CONFIG_CUSTOM_LOG_DIRECTIVE set_to_update = self.access_paths - match = pattern.match(line) - if not match: + if not (match := pattern.match(line)): self.target.log.warning("Unexpected Apache 'ErrorLog' or 'CustomLog' configuration in %s: %r", path, line) return @@ -346,10 +338,8 @@ def _process_conf_line(self, path: Path, line: str) -> None: custom_log = self.target.fs.path(location) if env_var_match := RE_ENV_VAR_IN_STRING.match(location): - envvar_directive = env_var_match.groupdict() - env_var = envvar_directive["env_var"] - apache_log_dir = self.env_vars.get(env_var) - if apache_log_dir is None: + env_var = env_var_match.groupdict()["env_var"] + if (apache_log_dir := self.env_vars.get(env_var)) is None: self.target.log.warning("%s does not exist, cannot resolve '%s' in %s", env_var, custom_log, path) return @@ -358,7 +348,7 @@ def _process_conf_line(self, path: Path, line: str) -> None: set_to_update.update(path for path in custom_log.parent.glob(f"*{custom_log.name}*")) @cached_property - def env_vars(self) -> dict: + def env_vars(self) -> dict[str, str]: variables = {} for envvar_file in self.DEFAULT_ENVVAR_PATHS: if (file := self.target.fs.path(envvar_file)).exists(): From c4300fd9be797b69c3cd75cec8a11a543b22697c Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:52:40 +0100 Subject: [PATCH 14/16] support \r\n too --- dissect/target/plugins/apps/webserver/apache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 81118ddd45..56c80f1644 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -352,7 +352,8 @@ def env_vars(self) -> dict[str, str]: variables = {} for envvar_file in self.DEFAULT_ENVVAR_PATHS: if (file := self.target.fs.path(envvar_file)).exists(): - for line in file.read_text().split("\n"): + for line in file.open("rt").readlines(): + line = line.strip() key, _, value = line.partition("=") if key: variables[key.replace("export ", "")] = value.strip("\"'") From f0ed6867ff6bb52bab6d34e7b6a477d72394b362 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:10:10 +0100 Subject: [PATCH 15/16] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/target/plugins/apps/webserver/apache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 56c80f1644..f90e1f152f 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -352,9 +352,8 @@ def env_vars(self) -> dict[str, str]: variables = {} for envvar_file in self.DEFAULT_ENVVAR_PATHS: if (file := self.target.fs.path(envvar_file)).exists(): - for line in file.open("rt").readlines(): - line = line.strip() - key, _, value = line.partition("=") + for line in file.read_text().splitlines(): + key, _, value = line.strip().partition("=") if key: variables[key.replace("export ", "")] = value.strip("\"'") return variables From aaf9e152527d6f66f0cb329a0928ddb4fb88d5a3 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 19 Mar 2025 13:51:37 +0100 Subject: [PATCH 16/16] implement review feedback --- dissect/target/plugins/apps/webserver/apache.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index f90e1f152f..a1948e3221 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -256,21 +256,19 @@ def find_logs(self) -> None: # Check default Apache configs for CustomLog or ErrorLog directives for config in self.DEFAULT_CONFIG_PATHS: if (path := self.target.fs.path(config)).exists() and path not in seen: - self._process_conf_file(path) - seen.add(path) + self._process_conf_file(path, seen) # Check all .conf files inside the server root if self.server_root: for path in self.server_root.rglob("*.conf"): if path not in seen: - self._process_conf_file(path) - seen.add(path) + self._process_conf_file(path, seen) def _process_conf_file(self, path: Path, seen: set[Path] | None = None) -> None: """Process an Apache ``.conf`` file for ``ServerRoot``, ``CustomLog``, ``Include`` and ``OptionalInclude`` directives. Populates ``self.access_paths`` and ``self.error_paths``. """ - seen = seen or set() + seen = set() if seen is None else seen if path in seen: self.target.log.warning("Detected recursion in Apache configuration, file already parsed: %s", path)