outbound: add backend and route metadata to errors (#2428)

PRs #2418 and #2419 add per-route and per-backend request timeouts
configured by the `OutboundPolicies` API to the `MatchedRoute` and
`MatchedBackend` layers in the outbound `ClientPolicy` stack,
respectively. This means that — unlike in the `ServiceProfile` stack —
two separate request timeouts can be configured in `ClientPolicy`
stacks. However, because both the `MatchedRoute` and `MatchedBackend`
layers are in the HTTP logical stack, the errors emitted by both
timeouts will have a `LogicalError` as their most specific error
metadata, meaning that the log messages and `l5d-proxy-error` headers
recorded for these timeouts do not indicate whether the timeout that
failed the request was the route request timeout or the backend request
timeout.

In order to ensure this information is recorded and exposed to the user,
this branch adds two new error wrapper types, one of which enriches an
error with a `RouteRef`'s metadata, and one of which enriches an error
with a `BackendRef`'s metadata. The `MatchedRoute` stack now wraps all
errors with `RouteRef` metadata, and the `MatchedBackend` stack wraps
errors with `BackendRef` metadata. This way, when the route timeout
fails a request, the error will include the route metadata, while when
the backend request timeout fails a request, the error will include both
the route and backend metadata.

Adding these new error wrappers also has the additional side benefit of
adding this metadata to errors returned by filters, allowing users to
distinguish between errors emitted by a filter on a route rule and
errors emitted by a per-backend filter. Also, any other errors emitted
lower in the stack for requests that are handled by a client policy
stack will now also include this metadata, which seems generally useful.

Example errors, taken from a proxy unit test:

backend request:
```
logical service logical.test.svc.cluster.local:666: route httproute.test.timeout-route: backend service.test.test-svc:666: HTTP response timeout after 1s
```
route request:
```
logical service logical.test.svc.cluster.local:666: route httproute.test.timeout-route: HTTP response timeout after 2s
```

Author: hawkw

linkerd/app/outbound/src/http/concrete.rs

@@ -42,7 +42,7 @@ pub struct DispatcherFailed(Arc<str>);
 
 /// Wraps errors encountered in this module.
 #[derive(Debug, thiserror::Error)]
-#[error("{} {}.{}: {source}", backend.0.kind(), backend.0.name(), backend.0.namespace())]
+#[error("{}: {source}", backend.0)]
 pub struct BalanceError {
     backend: BackendRef,
     #[source]

linkerd/app/outbound/src/http/logical/policy/route.rs

@@ -50,6 +50,15 @@ pub(crate) type Grpc<T> = MatchedRoute<
 pub(crate) type BackendDistribution<T, F> = distribute::Distribution<Backend<T, F>>;
 pub(crate) type NewDistribute<T, F, N> = distribute::NewDistribute<Backend<T, F>, (), N>;
 
+/// Wraps errors with route metadata.
+#[derive(Debug, thiserror::Error)]
+#[error("route {}: {source}", route.0)]
+struct RouteError {
+    route: RouteRef,
+    #[source]
+    source: Error,
+}
+
 // === impl MatchedRoute ===
 
 impl<T, M, F, E> MatchedRoute<T, M, F, E>
@@ -115,6 +124,15 @@ where
                 // Sets an optional request timeout.
                 .push(http::NewTimeout::layer())
                 .push(classify::NewClassify::layer())
+                .push(svc::NewMapErr::layer_with(|rt: &Self| {
+                    let route = rt.params.route_ref.clone();
+                    move |source| {
+                        Error::from(RouteError {
+                            route: route.clone(),
+                            source,
+                        })
+                    }
+                }))
                 .push(svc::ArcNewService::layer())
                 .into_inner()
         })

linkerd/app/outbound/src/http/logical/policy/route/backend.rs

@@ -1,5 +1,5 @@
 use super::{super::Concrete, filters};
-use crate::RouteRef;
+use crate::{BackendRef, RouteRef};
 use linkerd_app_core::{proxy::http, svc, Error, Result};
 use linkerd_http_route as http_route;
 use linkerd_proxy_client_policy as policy;
@@ -30,6 +30,15 @@ pub struct ExtractMetrics {
     metrics: RouteBackendMetrics,
 }
 
+/// Wraps errors with backend metadata.
+#[derive(Debug, thiserror::Error)]
+#[error("backend {}: {source}", backend.0)]
+struct BackendError {
+    backend: BackendRef,
+    #[source]
+    source: Error,
+}
+
 // === impl Backend ===
 
 impl<T: Clone, F> Clone for Backend<T, F> {
@@ -113,6 +122,15 @@ where
                 .push(count_reqs::NewCountRequests::layer_via(ExtractMetrics {
                     metrics: metrics.clone(),
                 }))
+                .push(svc::NewMapErr::layer_with(|t: &Self| {
+                    let backend = t.params.concrete.backend_ref.clone();
+                    move |source| {
+                        Error::from(BackendError {
+                            backend: backend.clone(),
+                            source,
+                        })
+                    }
+                }))
                 .push(svc::ArcNewService::layer())
                 .into_inner()
         })

linkerd/proxy/client-policy/src/lib.rs

@@ -2,7 +2,7 @@
 #![forbid(unsafe_code)]
 
 use once_cell::sync::Lazy;
-use std::{borrow::Cow, hash::Hash, net::SocketAddr, num::NonZeroU16, sync::Arc, time};
+use std::{borrow::Cow, fmt, hash::Hash, net::SocketAddr, num::NonZeroU16, sync::Arc, time};
 
 pub mod grpc;
 pub mod http;
@@ -264,6 +264,27 @@ impl std::hash::Hash for Meta {
     }
 }
 
+impl fmt::Display for Meta {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Default { name } => write!(f, "default.{name}"),
+            Self::Resource {
+                kind,
+                name,
+                namespace,
+                port,
+                ..
+            } => {
+                write!(f, "{kind}.{namespace}.{name}")?;
+                if let Some(port) = port {
+                    write!(f, ":{port}")?
+                }
+                Ok(())
+            }
+        }
+    }
+}
+
 // === impl FailureAccrual ===
 
 impl Default for FailureAccrual {