diff --git a/.chloggen/audit-logs-add-more-fields.yaml b/.chloggen/audit-logs-add-more-fields.yaml new file mode 100644 index 0000000000000..9649935bb42b3 --- /dev/null +++ b/.chloggen/audit-logs-add-more-fields.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: googlecloudlogentry_encoding + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for request attributes and destination attributes in cloud audit logs + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [42160] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/extension/encoding/googlecloudlogentryencodingextension/README.md b/extension/encoding/googlecloudlogentryencodingextension/README.md index 9f4a31d9eff52..f0b06cc67b118 100644 --- a/extension/encoding/googlecloudlogentryencodingextension/README.md +++ b/extension/encoding/googlecloudlogentryencodingextension/README.md @@ -150,8 +150,27 @@ See the struct of the Cloud Audit Log payload in [AuditLog](https://cloud.google | `requestMetadata.callerIp` | `client.address` | | `requestMetadata.callerSuppliedUserAgent` | `user_agent.original` | | `requestMetadata.callerNetwork` | `gcp.audit.request.caller.network` | -| `requestMetadata.requestAttributes` | _Currently not supported_ | -| `requestMetadata.destinationAttributes` | _Currently not supported_ | +| `requestMetadata.requestAttributes.id` | `http.request.id` | +| `requestMetadata.requestAttributes.method` | `http.request.method` | +| `requestMetadata.requestAttributes.headers` | `http.request.header.
` | +| `requestMetadata.requestAttributes.path` | `url.path` | +| `requestMetadata.requestAttributes.host` | `http.request.header.host` | +| `requestMetadata.requestAttributes.scheme` | `url.scheme` | +| `requestMetadata.requestAttributes.query` | `url.query` | +| `requestMetadata.requestAttributes.time` | `gcp.audit.request.time` | +| `requestMetadata.requestAttributes.size` | `http.request.size` | +| `requestMetadata.requestAttributes.protocol` | `network.protocol.name` | +| `requestMetadata.requestAttributes.reason` | `gcp.audit.request.reason` | +| `requestMetadata.requestAttributes.auth.principal` | `gcp.audit.request.auth.principal` | +| `requestMetadata.requestAttributes.auth.audiences` | `gcp.audit.request.auth.audiences` | +| `requestMetadata.requestAttributes.auth.presenter` | `gcp.audit.request.auth.presenter` | +| `requestMetadata.requestAttributes.auth.accessLevels` | `gcp.audit.request.auth.access_levels` | +| `requestMetadata.requestAttributes.auth.claims` | _Currently not supported_ | +| `requestMetadata.destinationAttributes.ip` | `server.address` | +| `requestMetadata.destinationAttributes.port` | `server.port` | +| `requestMetadata.destinationAttributes.labels` | `gcp.audit.destination.label.` | +| `requestMetadata.destinationAttributes.principal` | `gcp.audit.destination.principal` | +| `requestMetadata.destinationAttributes.regionCode` | `gcp.audit.destination.region_code` | | `request` | _Currently not supported_ | | `response` | _Currently not supported_ | | `metadata` | _Currently not supported_ | diff --git a/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser.go b/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser.go index b2601dd619b62..a8e57874079c0 100644 --- a/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser.go +++ b/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser.go @@ -6,6 +6,7 @@ package auditlog // import "github.com/open-telemetry/opentelemetry-collector-co import ( "errors" "fmt" + "strings" gojson "github.com/goccy/go-json" "github.com/iancoleman/strcase" @@ -54,6 +55,28 @@ const ( // gcpAuditRequestCallerNetwork holds the network of the request caller gcpAuditRequestCallerNetwork = "gcp.audit.request.caller.network" + // gcpAuditRequestReason holds the reason for the request. + gcpAuditRequestReason = "gcp.audit.request.reason" + // gcpAuditRequestTime holds the timestamp for when the destination receives the last byte of the request + gcpAuditRequestTime = "gcp.audit.request.time" + // httpRequestID will hold the request ID from requestMetadata.requestAttributes.id + httpRequestID = "http.request.id" + // gcpAuditRequestAuthPrincipal holds the principal at transport level layer + gcpAuditRequestAuthPrincipal = "gcp.audit.request.auth.principal" + // gcpAuditRequestAuthAudiences holds the audience at transport level layer + gcpAuditRequestAuthAudiences = "gcp.audit.request.auth.audiences" + // gcpAuditRequestAuthPresenter holds the presenter at transport level layer + gcpAuditRequestAuthPresenter = "gcp.audit.request.auth.presenter" + // gcpAuditRequestAuthAccessLevels holds the list of access level resource names that allow + // resources to be accessed, at transport level layer + gcpAuditRequestAuthAccessLevels = "gcp.audit.request.auth.access_levels" + + // gcpAuditDestinationLabels holds the labels in the destination attributes + gcpAuditDestinationLabels = "gcp.audit.destination.label" + // gcpAuditDestinationPrincipal holds the identity of the destination + gcpAuditDestinationPrincipal = "gcp.audit.destination.principal" + // gcpAuditDestinationRegionCode holds the region code of the destination + gcpAuditDestinationRegionCode = "gcp.audit.destination.region_code" // gcpAuditPolicyViolationResourceType holds the esource type that the orgpolicy is checked against gcpAuditPolicyViolationResourceType = "gcp.audit.policy_violation.resource.type" @@ -145,11 +168,42 @@ type violationInfo struct { } type requestMetadata struct { - CallerIP string `json:"callerIp"` - CallerSuppliedUserAgent string `json:"callerSuppliedUserAgent"` - CallerNetwork string `json:"callerNetwork"` - // TODO Add requestAttributes - // TODO Add destinationAttributes + CallerIP string `json:"callerIp"` + CallerSuppliedUserAgent string `json:"callerSuppliedUserAgent"` + CallerNetwork string `json:"callerNetwork"` + RequestAttributes *requestAttributes `json:"requestAttributes"` + DestinationAttributes *destinationAttributes `json:"destinationAttributes"` +} + +type requestAttributes struct { + ID string `json:"id"` + Method string `json:"method"` + Headers map[string]string `json:"headers"` + Path string `json:"path"` + Host string `json:"host"` + Scheme string `json:"scheme"` + Query string `json:"query"` + Time string `json:"time"` + Size string `json:"size"` + Protocol string `json:"protocol"` + Reason string `json:"reason"` + Auth auth `json:"auth"` +} + +type auth struct { + Principal string `json:"principal"` + Audiences []string `json:"audiences"` + Presenter string `json:"presenter"` + AccessLevels []string `json:"accessLevels"` + // TODO Add support for claims +} + +type destinationAttributes struct { + IP string `json:"ip"` + Port string `json:"port"` + Labels map[string]string `json:"labels"` + Principal string `json:"principal"` + RegionCode string `json:"regionCode"` } // isValid checks that the log meets requirements @@ -215,11 +269,11 @@ func handleAuthorizationInfo(info []authorizationInfo, attr pcommon.Map) { } infoList := attr.PutEmptySlice(gcpAuditAuthorization) - for _, auth := range info { + for _, authI := range info { m := infoList.AppendEmpty().SetEmptyMap() - shared.PutStr(gcpAuditAuthorizationPermission, auth.Permission, m) - shared.PutBool(gcpAuditAuthorizationGranted, auth.Granted, m) - shared.PutStr(gcpAuditAuthorizationResource, auth.Resource, m) + shared.PutStr(gcpAuditAuthorizationPermission, authI.Permission, m) + shared.PutBool(gcpAuditAuthorizationGranted, authI.Granted, m) + shared.PutStr(gcpAuditAuthorizationResource, authI.Resource, m) } } @@ -258,14 +312,65 @@ func handlePolicyViolationInfo(info *policyViolationInfo, attr pcommon.Map) { } } -func handleRequestMetadata(metadata *requestMetadata, attr pcommon.Map) { +func handleRequestMetadata(metadata *requestMetadata, attr pcommon.Map) error { if metadata == nil { - return + return nil } shared.PutStr(string(semconv.ClientAddressKey), metadata.CallerIP, attr) shared.PutStr(string(semconv.UserAgentOriginalKey), metadata.CallerSuppliedUserAgent, attr) shared.PutStr(gcpAuditRequestCallerNetwork, metadata.CallerNetwork, attr) + + if metadata.RequestAttributes != nil { + if err := shared.AddStrAsInt(string(semconv.HTTPRequestSizeKey), metadata.RequestAttributes.Size, attr); err != nil { + return fmt.Errorf("failed to add http request size %s: %w", metadata.RequestAttributes.Size, err) + } + shared.PutStr(string(semconv.HTTPRequestMethodKey), metadata.RequestAttributes.Method, attr) + shared.PutStr(string(semconv.URLQueryKey), metadata.RequestAttributes.Query, attr) + shared.PutStr(string(semconv.URLPathKey), metadata.RequestAttributes.Path, attr) + shared.PutStr(string(semconv.URLSchemeKey), metadata.RequestAttributes.Scheme, attr) + shared.PutStr(gcpAuditRequestTime, metadata.RequestAttributes.Time, attr) + shared.PutStr("http.request.header.host", metadata.RequestAttributes.Host, attr) + for h, v := range metadata.RequestAttributes.Headers { + shared.PutStr("http.request.header."+strings.ToLower(h), v, attr) + } + shared.PutStr(string(semconv.NetworkProtocolNameKey), strings.ToLower(metadata.RequestAttributes.Protocol), attr) + shared.PutStr(gcpAuditRequestReason, metadata.RequestAttributes.Reason, attr) + shared.PutStr(httpRequestID, metadata.RequestAttributes.ID, attr) + shared.PutStr(gcpAuditRequestAuthPrincipal, metadata.RequestAttributes.Auth.Principal, attr) + shared.PutStr(gcpAuditRequestAuthPresenter, metadata.RequestAttributes.Auth.Presenter, attr) + if len(metadata.RequestAttributes.Auth.AccessLevels) > 0 { + sl := attr.PutEmptySlice(gcpAuditRequestAuthAccessLevels) + for _, level := range metadata.RequestAttributes.Auth.AccessLevels { + v := sl.AppendEmpty() + v.SetStr(level) + } + } + if len(metadata.RequestAttributes.Auth.Audiences) > 0 { + sl := attr.PutEmptySlice(gcpAuditRequestAuthAudiences) + for _, audience := range metadata.RequestAttributes.Auth.Audiences { + v := sl.AppendEmpty() + v.SetStr(audience) + } + } + } + + if metadata.DestinationAttributes != nil { + if err := shared.AddStrAsInt(string(semconv.ServerPortKey), metadata.DestinationAttributes.Port, attr); err != nil { + return fmt.Errorf("failed to add destination port %s: %w", metadata.DestinationAttributes.Port, err) + } + shared.PutStr(string(semconv.ServerAddressKey), metadata.DestinationAttributes.IP, attr) + shared.PutStr(gcpAuditDestinationPrincipal, metadata.DestinationAttributes.Principal, attr) + shared.PutStr(gcpAuditDestinationRegionCode, metadata.DestinationAttributes.RegionCode, attr) + if len(metadata.DestinationAttributes.Labels) > 0 { + m := attr.PutEmptyMap(gcpAuditDestinationLabels) + for l, v := range metadata.DestinationAttributes.Labels { + shared.PutStr(strcase.ToSnakeWithIgnore(l, "."), v, m) + } + } + } + + return nil } func ParsePayloadIntoAttributes(payload []byte, attr pcommon.Map) error { @@ -284,6 +389,10 @@ func ParsePayloadIntoAttributes(payload []byte, attr pcommon.Map) error { return fmt.Errorf("failed to add number of response items: %w", err) } + if err := handleRequestMetadata(log.RequestMetadata, attr); err != nil { + return fmt.Errorf("failed to add request metadata: %w", err) + } + shared.PutStr(gcpAuditResourceName, log.ResourceName, attr) handleResourceLocation(log.ResourceLocation, attr) @@ -291,7 +400,6 @@ func ParsePayloadIntoAttributes(payload []byte, attr pcommon.Map) error { handleAuthenticationInfo(log.AuthenticationInfo, attr) handleAuthorizationInfo(log.AuthorizationInfo, attr) handlePolicyViolationInfo(log.PolicyViolationInfo, attr) - handleRequestMetadata(log.RequestMetadata, attr) handlePolicyViolationInfo(log.PolicyViolationInfo, attr) return nil diff --git a/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser_test.go b/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser_test.go index 3f704f07b4c36..573089a13f6cc 100644 --- a/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser_test.go +++ b/extension/encoding/googlecloudlogentryencodingextension/internal/auditlog/parser_test.go @@ -239,6 +239,7 @@ func TestHandleRequestMetadata(t *testing.T) { tests := map[string]struct { metadata *requestMetadata expectedAttr map[string]any + expectsErr string }{ "nil": { metadata: nil, @@ -256,13 +257,102 @@ func TestHandleRequestMetadata(t *testing.T) { gcpAuditRequestCallerNetwork: "//compute.googleapis.com/projects/elastic-apps-163815/global/networks/__unknown__", }, }, + "request attributes": { + metadata: &requestMetadata{ + RequestAttributes: &requestAttributes{ + ID: "req-12345", + Method: "GET", + Headers: map[string]string{ + "User-Agent": "test-client/1.0", + "Accept": "application/json", + }, + Path: "/test/path", + Host: "example.com", + Scheme: "https", + Query: "foo=bar&baz=qux", + Time: "2025-08-21T12:34:56Z", + Size: "1234", + Protocol: "HTTP/1.1", + Reason: "test-reason", + Auth: auth{ + Principal: "user@example.com", + Audiences: []string{"test-service", "another-service"}, + Presenter: "test-presenter", + AccessLevels: []string{"level1", "level2"}, + }, + }, + }, + expectedAttr: map[string]any{ + string(semconv.HTTPRequestSizeKey): int64(1234), + string(semconv.HTTPRequestMethodKey): "GET", + string(semconv.URLQueryKey): "foo=bar&baz=qux", + string(semconv.URLPathKey): "/test/path", + string(semconv.URLSchemeKey): "https", + gcpAuditRequestTime: "2025-08-21T12:34:56Z", + "http.request.header.host": "example.com", + "http.request.header.user-agent": "test-client/1.0", + "http.request.header.accept": "application/json", + string(semconv.NetworkProtocolNameKey): "http/1.1", + gcpAuditRequestReason: "test-reason", + httpRequestID: "req-12345", + gcpAuditRequestAuthPrincipal: "user@example.com", + gcpAuditRequestAuthPresenter: "test-presenter", + gcpAuditRequestAuthAccessLevels: []any{"level1", "level2"}, + gcpAuditRequestAuthAudiences: []any{"test-service", "another-service"}, + }, + }, + "request attributes - invalid request size format": { + metadata: &requestMetadata{ + RequestAttributes: &requestAttributes{ + Size: "invalid", + }, + }, + expectsErr: "failed to add http request size", + }, + "destination attributes": { + metadata: &requestMetadata{ + DestinationAttributes: &destinationAttributes{ + IP: "10.0.0.1", + Port: "8080", + Principal: "serviceAccount:my-svc@project.iam.gserviceaccount.com", + RegionCode: "us-central1", + Labels: map[string]string{ + "env": "staging", + "team.owner": "devops", + }, + }, + }, + expectedAttr: map[string]any{ + string(semconv.ServerPortKey): int64(8080), + string(semconv.ServerAddressKey): "10.0.0.1", + gcpAuditDestinationPrincipal: "serviceAccount:my-svc@project.iam.gserviceaccount.com", + gcpAuditDestinationRegionCode: "us-central1", + gcpAuditDestinationLabels: map[string]any{ + "env": "staging", + "team.owner": "devops", + }, + }, + }, + "destination attributes - invalid port format": { + metadata: &requestMetadata{ + DestinationAttributes: &destinationAttributes{ + Port: "invalid", + }, + }, + expectsErr: "failed to add destination port", + }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { t.Parallel() attr := pcommon.NewMap() - handleRequestMetadata(tt.metadata, attr) + err := handleRequestMetadata(tt.metadata, attr) + if tt.expectsErr != "" { + require.ErrorContains(t, err, tt.expectsErr) + return + } + require.NoError(t, err) require.Equal(t, tt.expectedAttr, attr.AsRaw()) }) }