https://signoz.io logo
#support
Title
# support
p

Phanideep Tirumalasetty

10/17/2023, 6:59 PM
Hii all, i am facing some challenges in parsing the application logs using OTEL collector and pushing to signoz collector. Getting below error.Can some one help on this ################# error helper/transformer.go:110 Failed to process entry {"kind": "receiver", "name": "filelog", "operator_id": "custom-log-parser", "operator_type": "regex_parser", "error": "regex pattern does not match", "action": "send", "entry": {"observed_timestamp":"2023-10-17T185426.22747813Z","timestamp":"2023-10-18T002424.506866516+05:30","body":"2023-10-18T002424.506866516+05:30 stdout P 600\"},{\"id\":\"n528057826\ #################### below is my configmap
Copy code
data:
  config.yaml: |-
    receivers:
      filelog:
        include:
          - /var/log/pods/*/*/*.log
        exclude:
          # Exclude logs from all containers named otel-collector
          - /var/log/pods/*/otel-collector/*.log
          - /var/log/pods/sumologic*/*/*.log
          - /var/log/pods/fluent*/*/*.log
        start_at: end
        include_file_path: true
        include_file_name: false
        operators:
          # Find out which format is used by kubernetes
          - type: router
            id: get-format
            routes:
              - output: parser-docker
                expr: 'body matches "^\\{"'
              - output: parser-crio
                expr: 'body matches "^[^ Z]+ "'
              - output: parser-containerd
                expr: 'body matches "^[^ Z]+Z"'
              - output: custom-log-parser
                expr: 'body matches "ResponseLogger"'
          # Parse CRI-O format
          - type: regex_parser
            id: parser-crio
            regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
            output: extract_metadata_from_filepath
            timestamp:
              parse_from: attributes.time
              layout_type: gotime
              layout: '2006-01-02T15:04:05.999999999Z07:00'
          # Parse CRI-Containerd format
          - type: regex_parser
            id: parser-containerd
            regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
            output: extract_metadata_from_filepath
            timestamp:
              parse_from: attributes.time
              layout: '%Y-%m-%dT%H:%M:%S.%LZ'

          #recombine logs
          - type: recombine
            combine_field: body.message
            combine_with: ""
            is_last_entry: "body.logtag == 'F'"
            overwrite_with: "newest"

          # Parse Docker format
          - type: json_parser
            id: parser-docker
            output: extract_metadata_from_filepath
            timestamp:
              parse_from: attributes.time
              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
          # Extract metadata from file path
          - type: regex_parser
            id: extract_metadata_from_filepath
            regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
            parse_from: attributes["log.file.path"]
            cache:
              size: 128  # default maximum amount of Pods per Node is 110

          - type: regex_parser
            id: custom-log-parser
            regex: '^(?P<Date>([^ ]+)) (?P<Time>([^ ]+)) \[(?P<loglevel>([^ ]+)) ] (?P<thread>([^ ])) (?P<class>([^ ])) (?P<clientid>([^ ])) (?P<userid>([^ ])) (?P<responseid>([^ ])) (?P<appid>([^ ])) (?P<message>([^*]+))'
            timestamp:
               parse_from: attributes.Date
               layout: '%Y-%m-%dT%H:%M:%S.%LZ'

          # Update body field after finishing all parsing
          - type: move
            from: attributes.log
            to: body
          # Rename attributes
          - type: move
            from: attributes.stream
            to: attributes["log.iostream"]
          - type: move
            from: attributes.container_name
            to: resource["k8s.container.name"]
          - type: move
            from: attributes.namespace
            to: resource["k8s.namespace.name"]
          - type: move
            from: attributes.pod_name
            to: resource["k8s.pod.name"]
          - type: move
            from: attributes.restart_count
            to: resource["k8s.container.restart_count"]
          - type: move
            from: attributes.uid
            to: resource["k8s.pod.uid"]

    processors:
      # k8sattributes processor to get the metadata from K8s
      k8sattributes:
        auth_type: "serviceAccount"
        passthrough: false
        extract:
          metadata:
            - k8s.pod.name
            - k8s.pod.uid
            - k8s.deployment.name
            - k8s.namespace.name
            - k8s.node.name
            - k8s.pod.start_time
          # Pod labels which can be fetched via K8sattributeprocessor
          labels:
            - tag_name: key1
              key: label1
              from: pod
            - tag_name: key2
              key: label2
              from: pod
        # Pod association using resource attributes and connection
        pod_association:
          - from: resource_attribute
            name: k8s.pod.uid
          - from: resource_attribute
            name: k8s.pod.ip
          - from: connection

    exporters:
      otlp:
        endpoint: "XX.XX.XX.XX:4317"
        tls:
          insecure: true

    service:
      pipelines:
        logs:
          receivers: [filelog]
          processors: [k8sattributes]
          exporters: [otlp]