feat(auto_source_config): Derive in-app stack trace rules (#87842)

This is the logic to mark frames as in-app when the files are found in
the developers' GitHub repositories.

This is very important for languages where the SDK or our internal rules
cannot determine which frames are in-app and which are not.

The initial language to get support for this is Java.

This will run in dry run mode to discover errors or scaling problems.

---------

Co-authored-by: Katie Byers <[email protected]>

Author: armenzg

src/sentry/grouping/api.py

@@ -37,6 +37,7 @@
     HashedChecksumVariant,
     SaltedComponentVariant,
 )
+from sentry.issues.auto_source_code_config.constants import DERIVED_ENHANCEMENTS_OPTION_KEY
 from sentry.models.grouphash import GroupHash
 
 if TYPE_CHECKING:
@@ -87,6 +88,7 @@ def get_config_dict(self, project: Project) -> GroupingConfig:
         }
 
     def _get_enhancements(self, project: Project) -> str:
+        derived_enhancements = project.get_option(DERIVED_ENHANCEMENTS_OPTION_KEY)
         project_enhancements = project.get_option("sentry:grouping_enhancements")
 
         config_id = self._get_config_id(project)
@@ -100,15 +102,27 @@ def _get_enhancements(self, project: Project) -> str:
         cache_prefix = self.cache_prefix
         cache_prefix += f"{LATEST_VERSION}:"
         cache_key = (
-            cache_prefix + md5_text(f"{enhancements_base}|{project_enhancements}").hexdigest()
+            cache_prefix
+            + md5_text(
+                f"{enhancements_base}|{derived_enhancements}|{project_enhancements}"
+            ).hexdigest()
         )
         enhancements = cache.get(cache_key)
         if enhancements is not None:
             return enhancements
 
         try:
+            # Automatic enhancements are always applied first, so they can be overridden by
+            # project-specific enhancements.
+            enhancements_string = project_enhancements or ""
+            if derived_enhancements:
+                enhancements_string = (
+                    f"{derived_enhancements}\n{enhancements_string}"
+                    if enhancements_string
+                    else derived_enhancements
+                )
             enhancements = Enhancements.from_config_string(
-                project_enhancements, bases=[enhancements_base] if enhancements_base else []
+                enhancements_string, bases=[enhancements_base] if enhancements_base else []
             ).base64_string
         except InvalidEnhancerConfig:
             enhancements = get_default_enhancements()
@@ -441,7 +455,7 @@ def get_grouping_variants_for_event(
 
 
 def get_contributing_variant_and_component(
-    variants: dict[str, BaseVariant]
+    variants: dict[str, BaseVariant],
 ) -> tuple[BaseVariant, ContributingComponent | None]:
     if len(variants) == 1:
         contributing_variant = list(variants.values())[0]

src/sentry/issues/auto_source_code_config/code_mapping.py

@@ -540,8 +540,8 @@ def find_roots(frame_filename: FrameInfo, source_path: str) -> tuple[str, str]:
     elif source_path.endswith(stack_path):  # "Packaged" logic
         source_prefix = source_path.rpartition(stack_path)[0]
         return (
-            f"{stack_root}{frame_filename.stack_root}/",
-            f"{source_prefix}{frame_filename.stack_root}/",
+            f"{stack_root}{frame_filename.stack_root}/".replace("//", "/"),
+            f"{source_prefix}{frame_filename.stack_root}/".replace("//", "/"),
         )
     elif stack_path.endswith(source_path):
         stack_prefix = stack_path.rpartition(source_path)[0]
@@ -593,9 +593,17 @@ def get_path_from_module(module: str, abs_path: str) -> tuple[str, str]:
     if "." not in module:
         raise DoesNotFollowJavaPackageNamingConvention
 
-    # If module has a dot, take everything before the last dot
-    # com.example.foo.Bar$InnerClass -> com/example/foo
-    stack_root = module.rsplit(".", 1)[0].replace(".", "/")
-    file_path = f"{stack_root}/{abs_path}"
+    parts = module.split(".")
+
+    if len(parts) > 2:
+        # com.example.foo.bar.Baz$InnerClass, Baz.kt ->
+        #    stack_root: com/example/
+        #    file_path:  com/example/foo/bar/Baz.kt
+        stack_root = "/".join(parts[:2])
+        file_path = "/".join(parts[:-1]) + "/" + abs_path
+    else:
+        # a.Bar, Bar.kt -> stack_root: a/, file_path:  a/Bar.kt
+        stack_root = parts[0] + "/"
+        file_path = f"{stack_root}{abs_path}"
 
     return stack_root, file_path

src/sentry/issues/auto_source_code_config/constants.py

@@ -4,6 +4,7 @@
 from typing import Any
 
 METRIC_PREFIX = "auto_source_code_config"
+DERIVED_ENHANCEMENTS_OPTION_KEY = "sentry:derived_grouping_enhancements"
 SUPPORTED_INTEGRATIONS = ["github"]
 
 # Any new languages should also require updating the stacktraceLink.tsx

src/sentry/issues/auto_source_code_config/in_app_stack_trace_rules.py

@@ -0,0 +1,64 @@
+import logging
+from collections.abc import Sequence
+
+from sentry.issues.auto_source_code_config.code_mapping import CodeMapping
+from sentry.issues.auto_source_code_config.utils import PlatformConfig
+from sentry.models.project import Project
+from sentry.utils import metrics
+
+from .constants import DERIVED_ENHANCEMENTS_OPTION_KEY, METRIC_PREFIX
+
+logger = logging.getLogger(__name__)
+
+
+def save_in_app_stack_trace_rules(
+    project: Project, code_mappings: Sequence[CodeMapping], platform_config: PlatformConfig
+) -> list[str]:
+    """Save in app stack trace rules for a given project"""
+    rules_from_code_mappings = set()
+    for code_mapping in code_mappings:
+        try:
+            rules_from_code_mappings.add(generate_rule_for_code_mapping(code_mapping))
+        except ValueError:
+            pass
+
+    current_enhancements = project.get_option(DERIVED_ENHANCEMENTS_OPTION_KEY)
+    current_rules = set(current_enhancements.split("\n")) if current_enhancements else set()
+
+    united_rules = rules_from_code_mappings.union(current_rules)
+    if not platform_config.is_dry_run_platform() and united_rules != current_rules:
+        project.update_option(DERIVED_ENHANCEMENTS_OPTION_KEY, "\n".join(sorted(united_rules)))
+
+    new_rules_added = united_rules - current_rules
+    metrics.incr(
+        key=f"{METRIC_PREFIX}.in_app_stack_trace_rules.created",
+        amount=len(new_rules_added),
+        tags={
+            "platform": platform_config.platform,
+            "dry_run": platform_config.is_dry_run_platform(),
+        },
+        sample_rate=1.0,
+    )
+    return list(new_rules_added)
+
+
+# XXX: This is very Java specific. If we want to support other languages, we need to
+# come up with a better way to generate the rule.
+def generate_rule_for_code_mapping(code_mapping: CodeMapping) -> str:
+    """Generate an in-app rule for a given code mapping"""
+    stacktrace_root = code_mapping.stacktrace_root
+    if stacktrace_root == "":
+        raise ValueError("Stacktrace root is empty")
+
+    parts = stacktrace_root.rstrip("/").split("/", 2)
+    # We only want the first two parts
+    module = ".".join(parts[:2])
+
+    if module == "":
+        raise ValueError("Module is empty")
+
+    # a/ -> a.**
+    # x/y/ -> x.y.**
+    # com/example/foo/bar/ -> com.example.**
+    # uk/co/example/foo/bar/ -> uk.co.**
+    return f"stack.module:{module}.** +app"

src/sentry/issues/auto_source_code_config/task.py

@@ -25,6 +25,7 @@
 from sentry.utils.locking import UnableToAcquireLock
 
 from .constants import METRIC_PREFIX
+from .in_app_stack_trace_rules import save_in_app_stack_trace_rules
 from .integration_utils import (
     InstallationCannotGetTreesError,
     InstallationNotFoundError,
@@ -192,8 +193,9 @@ def create_configurations(
 
     in_app_stack_trace_rules: list[str] = []
     if platform_config.creates_in_app_stack_trace_rules():
-        # XXX: This will be changed on the next PR
-        pass
+        in_app_stack_trace_rules = save_in_app_stack_trace_rules(
+            project, code_mappings, platform_config
+        )
 
     # We return this to allow tests running in dry-run mode to assert
     # what would have been created.

src/sentry/models/options/project_option.py

@@ -53,6 +53,7 @@
         "sentry:grouping_enhancements_base",
         "sentry:secondary_grouping_config",
         "sentry:secondary_grouping_expiry",
+        "sentry:derived_grouping_enhancements",
         "sentry:similarity_backfill_completed",
         "sentry:fingerprinting_rules",
         "sentry:relay_pii_config",

src/sentry/projectoptions/defaults.py

@@ -30,6 +30,7 @@
 )
 
 register(key="sentry:grouping_enhancements", default="")
+register(key="sentry:derived_grouping_enhancements", default="")
 
 # server side fingerprinting defaults.
 register(key="sentry:fingerprinting_rules", default="")