Phanideep Tirumalasetty
10/17/2023, 6:59 PMdata:
config.yaml: |-
receivers:
filelog:
include:
- /var/log/pods/*/*/*.log
exclude:
# Exclude logs from all containers named otel-collector
- /var/log/pods/*/otel-collector/*.log
- /var/log/pods/sumologic*/*/*.log
- /var/log/pods/fluent*/*/*.log
start_at: end
include_file_path: true
include_file_name: false
operators:
# Find out which format is used by kubernetes
- type: router
id: get-format
routes:
- output: parser-docker
expr: 'body matches "^\\{"'
- output: parser-crio
expr: 'body matches "^[^ Z]+ "'
- output: parser-containerd
expr: 'body matches "^[^ Z]+Z"'
- output: custom-log-parser
expr: 'body matches "ResponseLogger"'
# Parse CRI-O format
- type: regex_parser
id: parser-crio
regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout_type: gotime
layout: '2006-01-02T15:04:05.999999999Z07:00'
# Parse CRI-Containerd format
- type: regex_parser
id: parser-containerd
regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
#recombine logs
- type: recombine
combine_field: body.message
combine_with: ""
is_last_entry: "body.logtag == 'F'"
overwrite_with: "newest"
# Parse Docker format
- type: json_parser
id: parser-docker
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
# Extract metadata from file path
- type: regex_parser
id: extract_metadata_from_filepath
regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
parse_from: attributes["log.file.path"]
cache:
size: 128 # default maximum amount of Pods per Node is 110
- type: regex_parser
id: custom-log-parser
regex: '^(?P<Date>([^ ]+)) (?P<Time>([^ ]+)) \[(?P<loglevel>([^ ]+)) ] (?P<thread>([^ ])) (?P<class>([^ ])) (?P<clientid>([^ ])) (?P<userid>([^ ])) (?P<responseid>([^ ])) (?P<appid>([^ ])) (?P<message>([^*]+))'
timestamp:
parse_from: attributes.Date
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
# Update body field after finishing all parsing
- type: move
from: attributes.log
to: body
# Rename attributes
- type: move
from: attributes.stream
to: attributes["log.iostream"]
- type: move
from: attributes.container_name
to: resource["k8s.container.name"]
- type: move
from: attributes.namespace
to: resource["k8s.namespace.name"]
- type: move
from: attributes.pod_name
to: resource["k8s.pod.name"]
- type: move
from: attributes.restart_count
to: resource["k8s.container.restart_count"]
- type: move
from: attributes.uid
to: resource["k8s.pod.uid"]
processors:
# k8sattributes processor to get the metadata from K8s
k8sattributes:
auth_type: "serviceAccount"
passthrough: false
extract:
metadata:
- k8s.pod.name
- k8s.pod.uid
- k8s.deployment.name
- k8s.namespace.name
- k8s.node.name
- k8s.pod.start_time
# Pod labels which can be fetched via K8sattributeprocessor
labels:
- tag_name: key1
key: label1
from: pod
- tag_name: key2
key: label2
from: pod
# Pod association using resource attributes and connection
pod_association:
- from: resource_attribute
name: k8s.pod.uid
- from: resource_attribute
name: k8s.pod.ip
- from: connection
exporters:
otlp:
endpoint: "XX.XX.XX.XX:4317"
tls:
insecure: true
service:
pipelines:
logs:
receivers: [filelog]
processors: [k8sattributes]
exporters: [otlp]