Skip to content

Commit 913f89a

Browse files
adnanrahicadnanrahic
committed
fix parsing k8s filenames
1 parent f7803d1 commit 913f89a

2 files changed

Lines changed: 98 additions & 21 deletions

File tree

‎config/examples/file-input-glob-patterns-es-output.yml‎

Lines changed: 70 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,81 @@
11
# Global options
22
options:
33
debug: true
4+
# maxLogSize: 16384
45

5-
input:
6-
files:
7-
# ship everything EXCEPT for auth*.log, system*.log, and kernel*.log
8-
- /var/log/!(auth*.log|system*.log|kernel*.log)
6+
# input:
7+
# files:
8+
# # ship everything EXCEPT for auth*.log, system*.log, and kernel*.log
9+
# - /var/log/!(auth*.log|system*.log|kernel*.log)
910

10-
# ship everything EXCEPT for kube*.log, storage*.log, etcd*.log, and coredns*.log
11-
- /var/log/containers/!(kube*.log|storage*.log|etcd*.log|coredns*.log)
11+
# # ship everything EXCEPT for kube*.log, storage*.log, etcd*.log, and coredns*.log
12+
# - /var/log/containers/!(kube*.log|storage*.log|etcd*.log|coredns*.log)
13+
14+
# inputFilter:
15+
# - module: grep
16+
# config:
17+
# matchSource: !!js/regexp /.*log/ # match log files
18+
# include: !!js/regexp /failed|error|exception/i # include errors
19+
# exclude: !!js/regexp /super noisy error messages/i # exclude noise
20+
21+
input:
22+
# stdin: true
23+
files:
24+
- '/home/raha/code/sematext/logagent-js/0.log'
1225

1326
inputFilter:
14-
- module: grep
15-
config:
16-
matchSource: !!js/regexp /.*log/ # match log files
17-
include: !!js/regexp /failed|error|exception/i # include errors
18-
exclude: !!js/regexp /super noisy error messages/i # exclude noise
27+
# parse containerd log format, add pod info to log context
28+
- module: input-filter-k8s-containerd
29+
# - module: input-filter-k8s-containerd-multiline
30+
31+
# outputFilter:
32+
# add k8s metadata via k8s API
33+
# - module: kubernetes-enrichment
34+
35+
# parser:
36+
# patterns:
37+
# -
38+
# # sourceName: !!js/regexp /\.log/ # catch all system.log files
39+
# sourceName: !!js/regexp /unknown/
40+
# # blockStart: !!js/regexp /\d{4}\-\d{2}\-\d{2}\T\d{2}:\d{2}:\d{2}\.\d{9}Z\sstdout\sP/
41+
# match:
42+
# - type: venkat_custom_log_partial
43+
# regex: !!js/regexp /\[(.*)\]\s\[(.*)\]\s\[(.*)\]\s*Parameters:\s(.*)/
44+
# fields:
45+
# - trace_id
46+
# - customer_name
47+
# - customer_id
48+
# - message
49+
# - type: venkat_custom_log
50+
# regex: !!js/regexp /(\d{4}\-\d{2}\-\d{2}\T\d{2}:\d{2}:\d{2}\.\d{9}Z)\sstdout\s(P|F)\s(.*)/
51+
# fields:
52+
# - ts
53+
# - stdout_type
54+
# - message
55+
# - type: venkat_custom_log_partial
56+
# regex: !!js/regexp /(\d{4}\-\d{2}\-\d{2}\T\d{2}:\d{2}:\d{2}\.\d{9}Z)\s(stdout\sP)\s\[(.*)\]\s\[(.*)\]\s\[(.*)\]\s*Parameters:\s(.*)/
57+
# fields:
58+
# - ts
59+
# - stdout_type
60+
# - trace_id
61+
# - customer_name
62+
# - customer_id
63+
# - message
64+
# - type: venkat_custom_log_full
65+
# regex: !!js/regexp /(\d{4}\-\d{2}\-\d{2}\T\d{2}:\d{2}:\d{2}\.\d{9}Z)\s(stdout\sF)\s\[(.*)\]\s\[(.*)\]\s\[(.*)\]\s*Parameters:\s(.*)/
66+
# fields:
67+
# - ts
68+
# - stdout_type
69+
# - trace_id
70+
# - customer_name
71+
# - customer_id
72+
# - message
1973

2074
output:
75+
# stdout: yaml
2176
elasticsearch:
2277
module: elasticsearch
23-
url: https://logsene-receiver.sematext.com
24-
index: YOUR_LOGS_APP_TOKEN
78+
url: https://logs-token-receiver.apps.test.sematext.com
79+
index: 174ac025-ea55-426d-b544-fcc3876ad247
80+
# maxBufferSize: 1
81+
# flushInterval: 10

‎lib/plugins/input-filter/kubernetesContainerd.js‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ const sources = {}
66
* data - the log message as string
77
* callback - callback function (err, data).
88
*/
9+
10+
// fileName: app-77b4d5595b-hmjxs_default_app-80209fd578c6be842b5b8a2d6389227ccab0196b7b658bd245eed4767c9b843c.log
11+
// meta: {
12+
// 0: app-77b4d5595b-hmjxs, // pod
13+
// 1: default // namespace
14+
// 2: app-80209fd578c6be842b5b8a2d6389227ccab0196b7b658bd245eed4767c9b843c.log // container with .log suffix
15+
// }
16+
917
function parseK8sFileName (sourceName) {
1018
// cut path from /var/log/containers/<pod_name>_<pod_namespace>_<container_name>-<container_id>.log
1119
// Reference: https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/kubelet-cri-logging.md
@@ -15,16 +23,28 @@ function parseK8sFileName (sourceName) {
1523
var info = {}
1624

1725
if (meta.length === 3) {
26+
const { 0: name, 1: namespace, 2: containerWithLogSuffix } = meta
1827
info.kubernetes = {
19-
pod: { name: meta[0] },
20-
namespace: meta[1]
28+
pod: {
29+
name,
30+
container: {}
31+
},
32+
namespace
33+
}
34+
35+
const positionOfNameAndIdSeparator = containerWithLogSuffix.lastIndexOf('-')
36+
const positionOfDotLog = containerWithLogSuffix.indexOf('.')
37+
const containerId = containerWithLogSuffix.substring(
38+
positionOfNameAndIdSeparator + 1,
39+
positionOfDotLog
40+
)
41+
const containerName = containerWithLogSuffix.substring(0, index)
42+
43+
if (containerId) {
44+
info.kubernetes.pod.container.id = containerId
2145
}
22-
index = meta[2].lastIndexOf('-')
23-
var endOfId = meta[2].indexOf('.')
24-
var containerName = meta[2].substring(index + 1, endOfId)
2546
if (containerName) {
26-
info.kubernetes.pod.container.name = meta[2].substring(0, index)
27-
info.kubernetes.pod.container.id = containerName
47+
info.kubernetes.pod.container.name = containerName
2848
}
2949
}
3050
return info
@@ -77,7 +97,7 @@ module.exports = function (context, config, data, callback) {
7797
sources[sourceName].previousStreamFlag === 'P'
7898
) {
7999
sources[sourceName].logLines.push(logLine)
80-
const joinedLogLine = sources[sourceName].logLines.join(' ')
100+
const joinedLogLine = sources[sourceName].logLines.join(' ')
81101
console.log('\n\n\n\n\n\n\n\n\n\n\n\n')
82102
console.log('FULL')
83103
console.log(sources[sourceName])

0 commit comments

Comments
 (0)