unable to parse below json log to signoz using the...
# support
n
unable to parse below json log to signoz using the below configuration file {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"slot_details","slotDetail":{"stm_id":"684178718","time":"143000","end_time":"153000","locality_id":0,"booking_id":0,"freeze_count":0,"date":"2024-02-07","user_id":0,"collection_center_id":0,"b2b_zone_id":0,"zone_id":53,"sample_type_id":1,"isavailable":1,"isactive":1,"set_name":"set5","is_peak_hours":0,"source":"crm","order_group_id":0,"old_zone_id":0,"samplecollector_id":0,"old_set_name":"","phlebo_name":"","assign_time":"0000-00-00 000000","freeze":0,"freeze_date":"0000-00-00 000000","payment":0,"locusStatus":0,"lat":"","lng":"","dropTaskStatus":0,"autoAssignStatus":0,"pickup_type":"home","service_id":0,"consultation_id":0,"channel_type":0,"channel_user":0,"vendor_user_id":"","created_at":"2024-02-05 125610","updated_at":"2024-02-05 125610","reference_stm_id":0,"phlebo_id":0},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2112,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 135812"} Configuration file : - receivers: filelog: include: [ "/var/log/newcrm/el-t25crm.healthians.co.in-2024-02-*.log" ] start_at: end include_file_path: true include_file_name: false poll_interval: 200ms operators: - type: regex_parser regex: '^.(?P<body>{.})' output: parsedata - type: json_parser id: parsedata parse_from: attributes.body - type: move from: attributes["body"] to: body - type: time_parser parse_from: attributes.time layout: '2006-01-02T150405Z07:00' layout_type: gotime processors: batch: send_batch_size: 10000 send_batch_max_size: 11000 timeout: 10s exporters: otlp/log: endpoint: http://slog001.echl.in:4317 tls: insecure: true service: pipelines: logs: receivers: [filelog] processors: [batch] exporters: [otlp/log] ~
r
Hi Nikhil 🙂 Please try using the regex
^.*?(?P<body>{.*})
- note the ? after .* - which makes it ungreedy You can see the diff between not having the ? and having it If you still have issues, please send your otel collector logs and some samples of the processed logs from SigNoz UI
n
thanks i will check and get back to you
Below are the sample log files that otel-collector need to send logs to signoz {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"Adding Post Data","post_data":{"customer":[{"relation":"others","age":28,"dob":null,"gender":"F","email":null,"customer_id":"66aff7d8-cd9b-4796-bf17-e66088c3565a","customer_name":"Prena","contact_number":null}],"slot":{"slot_id":"684178718"},"gender":"M","mobile":"6700670875","email":"siddhant.khanna_testi_blr+rahul@acko.tech","state":"26","cityId":"23","latitude":28.5101825,"longitude":77.0682661,"address":"sector 21, gurgaon, Gurgaon, Haryana, 122016","zipcode":"122016","landmark":"home","altmobile":false,"altemail":false,"package":[{"deal_id":["package_216"]}],"customer_calling_number":"6700670875","billing_cust_name":"Rahu","sub_locality":"sector 21, gurgaon, Gurgaon, Haryana, 122016","hard_copy":0,"vendor_booking_id":false,"vendor_billing_user_id":"4rWzYn5NOhEgNPdiVGWiRA","payment_option":"invoice","discounted_price":0,"sample_collected":"n","role_id":0,"search_by":"customer_mobile","channel_partner":"3","channel_partner_user":"5490","booked_source":"api","zip_search":2,"calling_from":"external_api","b2bNewExistingUser":false,"user_id_ajax":"","whatsappconcent":1,"billing_zone_id":"53","zone_id":"53","amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","billing_cust_city":"gurgaon","billing_cust_city_id":"23","locality":"541","channel_type":"3","channel_user":"5490"},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":1227,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 135812"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"Booking Form Validation","booking_form_validate_reponse":{"status":true,"message":"validate successfully.","resCode":"RES0001"},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":1233,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 135812"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"gross_margin","gross_margin_key":{"active":"on"},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2046,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 135812"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"slot_details","slotDetail":{"stm_id":"684178718","time":"143000","end_time":"153000","locality_id":0,"booking_id":0,"freeze_count":0,"date":"2024-02-07","user_id":0,"collection_center_id":0,"b2b_zone_id":0,"zone_id":53,"sample_type_id":1,"isavailable":1,"isactive":1,"set_name":"set5","is_peak_hours":0,"source":"crm","order_group_id":0,"old_zone_id":0,"samplecollector_id":0,"old_set_name":"","phlebo_name":"","assign_time":"0000-00-00 000000","freeze":0,"freeze_date":"0000-00-00 000000","payment":0,"locusStatus":0,"lat":"","lng":"","dropTaskStatus":0,"autoAssignStatus":0,"pickup_type":"home","service_id":0,"consultation_id":0,"channel_type":0,"channel_user":0,"vendor_user_id":"","created_at":"2024-02-05 125610","updated_at":"2024-02-05 125610","reference_stm_id":0,"phlebo_id":0},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2112,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 135812"} and below are the sample logs that are already coming in signoz { "attributes_bool": {}, "attributes_float64": {}, "attributes_int64": {}, "attributes_string": { "body": "{\"time\":\"2024-02-08T162907+05:30\",\"request_id\":\"cde731db9d09b894c46382f9f45050fe\",\"http_x_amzn_trace_id\":\"Root=1-65c4b3fb-35323611678cfe8754f895da\",\"http_x_forwarded_for\":\"182.74.128.251\",\"remote_addr\":\"182.74.128.251\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\"http_host\":\"erp.healthians.com\",\"request_method\":\"GET\",\"request_uri\":\"/manage/bookings\",\"status\":\"200\",\"request_time\":\"0.053\",\"request_length\":\"892\",\"body_bytes_sent\":\"12952\",\"upstream_addr\":\"unix:/run/php/php8.2-fpm.sock\",\"hostname\":\"erp003.echl.in\"}", "body_bytes_sent": "12952", "hostname": "erp003.echl.in", "http_host": "erp.healthians.com", "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", "http_x_amzn_trace_id": "Root=1-65c4b3fb-35323611678cfe8754f895da", "http_x_forwarded_for": "182.74.128.251", "log.file.path": "/var/log/nginx/erp.healthians.com.access.log", "remote_addr": "182.74.128.251", "request_id": "cde731db9d09b894c46382f9f45050fe", "request_length": "892", "request_method": "GET", "request_time": "0.053", "request_uri": "/manage/bookings", "status": "200", "time": "2024-02-08T162907+05:30", "upstream_addr": "unix:/run/php/php8.2-fpm.sock" }, "body": "{\"time\":\"2024-02-08T162907+05:30\",\"request_id\":\"cde731db9d09b894c46382f9f45050fe\",\"http_x_amzn_trace_id\":\"Root=1-65c4b3fb-35323611678cfe8754f895da\",\"http_x_forwarded_for\":\"182.74.128.251\",\"remote_addr\":\"182.74.128.251\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\"http_host\":\"erp.healthians.com\",\"request_method\":\"GET\",\"request_uri\":\"/manage/bookings\",\"status\":\"200\",\"request_time\":\"0.053\",\"request_length\":\"892\",\"body_bytes_sent\":\"12952\",\"upstream_addr\":\"unix:/run/php/php8.2-fpm.sock\",\"hostname\":\"erp003.echl.in\"}", "id": "2blH0qDloNxl3ksakEPmK2ekv5F", "resources_string": {}, "severity_number": 0, "severity_text": "", "span_id": "", "trace_flags": 0, "trace_id": "", "timestamp": "2024-02-08T105907Z" } { "attributes_bool": {}, "attributes_float64": {}, "attributes_int64": {}, "attributes_string": { "body": "{\"time\":\"2024-02-08T162907+05:30\",\"request_id\":\"958735cb01cd48374a67343958107b41\",\"http_x_amzn_trace_id\":\"Root=1-65c4b3fb-12f272295303dead3de5f108\",\"http_x_forwarded_for\":\"14.97.111.234\",\"remote_addr\":\"14.97.111.234\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\"http_host\":\"erp.healthians.com\",\"request_method\":\"POST\",\"request_uri\":\"/service/ticket_management/updateTicketStatus\",\"status\":\"200\",\"request_time\":\"0.327\",\"request_length\":\"2924\",\"body_bytes_sent\":\"95\",\"upstream_addr\":\"unix:/run/php/php8.2-fpm.sock\",\"hostname\":\"erp003.echl.in\"}", "body_bytes_sent": "95", "hostname": "erp003.echl.in", "http_host": "erp.healthians.com", "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", "http_x_amzn_trace_id": "Root=1-65c4b3fb-12f272295303dead3de5f108", "http_x_forwarded_for": "14.97.111.234", "log.file.path": "/var/log/nginx/erp.healthians.com.access.log", "remote_addr": "14.97.111.234", "request_id": "958735cb01cd48374a67343958107b41", "request_length": "2924", "request_method": "POST", "request_time": "0.327", "request_uri": "/service/ticket_management/updateTicketStatus", "status": "200", "time": "2024-02-08T162907+05:30", "upstream_addr": "unix:/run/php/php8.2-fpm.sock" }, "body": "{\"time\":\"2024-02-08T162907+05:30\",\"request_id\":\"958735cb01cd48374a67343958107b41\",\"http_x_amzn_trace_id\":\"Root=1-65c4b3fb-12f272295303dead3de5f108\",\"http_x_forwarded_for\":\"14.97.111.234\",\"remote_addr\":\"14.97.111.234\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\"http_host\":\"erp.healthians.com\",\"request_method\":\"POST\",\"request_uri\":\"/service/ticket_management/updateTicketStatus\",\"status\":\"200\",\"request_time\":\"0.327\",\"request_length\":\"2924\",\"body_bytes_sent\":\"95\",\"upstream_addr\":\"unix:/run/php/php8.2-fpm.sock\",\"hostname\":\"erp003.echl.in\"}", "id": "2blH0qDloNxl3ksakEPmK2ekv5D", "resources_string": {}, "severity_number": 0, "severity_text": "", "span_id": "", "trace_flags": 0, "trace_id": "", "timestamp": "2024-02-08T105907Z" }
below are the configuration file receivers: filelog: include: [ "/var/log/newcrm/el-t25crm.healthians.co.in-2024-02-*.log" ] start_at: beginning include_file_path: true include_file_name: false poll_interval: 200ms operators: - type: regex_parser regex: '^.(?P<message>{.})' output: parsedata - type: json_parser id: parsedata parse_from: attributes.message - type: move from: attributes["message"] to: body - type: time_parser parse_from: attributes.time layout: '%Y-%m-%d,%H:%M:%S %z' # layout_type: gotime processors: batch: send_batch_size: 10000 send_batch_max_size: 11000 timeout: 10s exporters: otlp/log: endpoint: http://slog001.echl.in:4317 tls: insecure: true service: pipelines: logs: receivers: [filelog] processors: [batch] exporters: [otlp/log]
i tried regex both ways '^.(?P<message>{.})' '^.?(?P<message>{.})'
but still getting below error error helper/transformer.go:98 Failed to process entry {"kind": "receiver", "name": "filelog", "data_type": "logs", "operator_id": "parsedata", "operator_type": "json_parser", "error": "Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|_logger\"}},\"amzn_tra|..., bigger context ...|ddProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400|...", "action": "send", "entry": {"observed_timestamp":"2024-02-08T162417.591024147+05:30","timestamp":"0001-01-01T000000Z","body":"{\"channel\":\"crm_process_booking_log\",\"log_type\":\"DEBUG\",\"index_name\":\"crm_process_booking_log\",\"index_type\":\"crm_process_booking_log\",\"error_type\":0,\"message\":{\"data\":{\"STEP\":\"Adding Post Data\",\"post_data\":{\"customer\":[{\"customer_id\":\"rwe18y79gxv5l6gz57yj4kod630qpm2z__716l2gd8ev9y4mg4ur176gov6z59nwkv\",\"customer_name\":\"Samarpit Sacha\",\"relation\":\"others\",\"age\":25,\"gender\":\"M\",\"contact_number\":\"8888822224\",\"email\":\"\"}],\"slot\":{\"slot_id\":\"684179245\"},\"package\":[{\"deal_id\":[\"profile_1\"]}],\"customer_calling_number\":\"8952954319\",\"billing_cust_name\":\"Rajesh QAC\",\"gender\":\"F\",\"mobile\":\"8952954319\",\"state\":\"26\",\"cityId\":\"23\",\"sub_locality\":\"vaishali nagar, jaipur\",\"latitude\":28.5100601,\"longitude\":77.0823482,\"address\":\"vaishali nagar, jaipur\",\"zipcode\":122016,\"altemail\":\"\",\"hard_copy\":0,\"vendor_booking_id\":\"TWC-974-38TKZX\",\"vendor_billing_user_id\":\"rwe18y79gxv5l6gz57yj4kod630qpm2z\",\"payment_option\":\"invoice\",\"discounted_price\":1199,\"sample_collected\":\"n\",\"role_id\":0,\"search_by\":\"customer_mobile\",\"channel_partner\":\"3\",\"channel_partner_user\":\"887\",\"booked_source\":\"api\",\"zip_search\":2,\"calling_from\":\"external_api\",\"b2bNewExistingUser\":false,\"user_id_ajax\":\"99797320352578\",\"billing_zone_id\":\"53\",\"zone_id\":\"53\",\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"billing_cust_city\":\"gurgaon\",\"billing_cust_city_id\":\"23\",\"locality\":\"614\",\"channel_type\":\"3\",\"channel_user\":\"887\"},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\"},\"calling_trace\":{\"file\":\"\\/var\\/www\\/newcrm-t25\\/application\\/controllers\\/crmuser\\/crmorder.php\",\"line\":1227,\"function\":\"addProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"nginx_request_id\":\"7cabecd57f1477ede470ea10c3ff450e\",\"timestamp\":\"2024-02-08 122622\"}","attributes":{"log.file.path":"/var/log/newcrm/el-t25crm.healthians.co.in-2024-02-08.log","message":"{\"file\":\"\\/var\\/www\\/newcrm-t25\\/application\\/controllers\\/crmuser\\/crmorder.php\",\"line\":1227,\"function\":\"addProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"nginx_request_id\":\"7cabecd57f1477ede470ea10c3ff450e\",\"timestamp\":\"2024-02-08 122622\"}"},"severity":0,"scope_name":""}} Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).HandleEntryError Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/transformer.go:98 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ParseWith Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:140 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:112 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:98 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/json.(*Parser).Process Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/parser/json/json.go:62 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/writer.go:53 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:122 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:98 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/regex.(*Parser).Process Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/parser/regex/regex.go:99 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/writer.go:53 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/file.(*Input).emit Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/input/file/file.go:52 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader.(*Reader).ReadToEnd Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/fileconsumer/internal/reader/reader.go:106 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer.(*Manager).consume.func1 Feb 08 162417 ast002.healthians.co.in otelcol-contrib[31314]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/fileconsumer/file.go:156
r
The error
Copy code
error        helper/transformer.go:98        Failed to process entry        {"kind": "receiver", "name": "filelog", "data_type": "logs", "operator_id": "parsedata", "operator_type": "json_parser", "error": "Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|_logger\"}},\"amzn_tra|...,
Would come if you take your first sample log below
Copy code
{
  "channel": "crm_process_booking_log",
  "log_type": "DEBUG",
  "index_name": "crm_process_booking_log",
  "index_type": "crm_process_booking_log",
  "error_type": 0,
  "message": {
    "data": {
      "STEP": "Adding Post Data",
      "post_data": {
        "customer": [
          {
            "relation": "others",
            "age": 28,
            "dob": null,
            "gender": "F",
            "email": null,
            "customer_id": "66aff7d8-cd9b-4796-bf17-e66088c3565a",
            "customer_name": "Prena",
            "contact_number": null
          }
        ],
        "slot": {
          "slot_id": "684178718"
        },
        "gender": "M",
        "mobile": "6700670875",
        "email": "siddhant.khanna_testi_blr+rahul@acko.tech",
        "state": "26",
        "cityId": "23",
        "latitude": 28.5101825,
        "longitude": 77.0682661,
        "address": "sector 21, gurgaon, Gurgaon, Haryana, 122016",
        "zipcode": "122016",
        "landmark": "home",
        "altmobile": false,
        "altemail": false,
        "package": [
          {
            "deal_id": [
              "package_216"
            ]
          }
        ],
        "customer_calling_number": "6700670875",
        "billing_cust_name": "Rahu",
        "sub_locality": "sector 21, gurgaon, Gurgaon, Haryana, 122016",
        "hard_copy": 0,
        "vendor_booking_id": false,
        "vendor_billing_user_id": "4rWzYn5NOhEgNPdiVGWiRA",
        "payment_option": "invoice",
        "discounted_price": 0,
        "sample_collected": "n",
        "role_id": 0,
        "search_by": "customer_mobile",
        "channel_partner": "3",
        "channel_partner_user": "5490",
        "booked_source": "api",
        "zip_search": 2,
        "calling_from": "external_api",
        "b2bNewExistingUser": false,
        "user_id_ajax": "",
        "whatsappconcent": 1,
        "billing_zone_id": "53",
        "zone_id": "53",
        "amzn_trace_id": "Root=1-65c33f1c-29fe535658001b076140677c",
        "billing_cust_city": "gurgaon",
        "billing_cust_city_id": "23",
        "locality": "541",
        "channel_type": "3",
        "channel_user": "5490"
      },
      "amzn_trace_id": "Root=1-65c33f1c-29fe535658001b076140677c"
    },
    "calling_trace": {
      "file": "/var/www/newcrm-t25/application/controllers/crmuser/crmorder.php",
      "line": 1227,
      "function": "addProcessBookingLog",
      "class": "Enterprise_logger"
    }
  },
  "amzn_trace_id": "Root=1-65c33f1c-29fe535658001b076140677c",
  "nginx_request_id": "c93176430784c9dd4e32e35541e6e31a",
  "timestamp": "2024-02-07 13:58:12"
}
and use the regex
^.*(?P<message>{.*})
(without ? after .*) The regex would match just the last part, leading to following value in
attributes.message
(as you can see at https://regex101.com/r/cjxTv9/1)
Copy code
{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":1227,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c33f1c-29fe535658001b076140677c","nginx_request_id":"c93176430784c9dd4e32e35541e6e31a","timestamp":"2024-02-07 13:58:12"}
And if you try JSON parsing that message, you would end up with the error in your otel collector Atleast your 1st sample log will not error out if you use the regex
^.*?(?P<message>{.*})
n
Still i am getting the same error .. i dont know where i am missing the configuration...Is there any link or any sample Configuration for that type of logs.. Kindly suggest error helper/transformer.go:98 Failed to process entry {"kind": "receiver", "name": "filelog", "data_type": "logs", "operator_id": "parsedata", "operator_type": "json_parser", "error": "Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|_logger\"}},\"amzn_tra|..., bigger context ...|ddProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400|...", "action": "send", "entry": {"observed_timestamp":"2024-02-09T115712.227373307+05:30","timestamp":"0001-01-01T000000Z","body":"{\"channel\":\"crm_process_booking_log\",\"log_type\":\"DEBUG\",\"index_name\":\"crm_process_booking_log\",\"index_type\":\"crm_process_booking_log\",\"error_type\":0,\"message\":{\"data\":{\"STEP\":\"Adding Post Data\",\"post_data\":{\"customer\":[{\"customer_id\":\"rwe18y79gxv5l6gz57yj4kod630qpm2z__716l2gd8ev9y4mg4ur176gov6z59nwkv\",\"customer_name\":\"Samarpit Sacha\",\"relation\":\"others\",\"age\":25,\"gender\":\"M\",\"contact_number\":\"8888822224\",\"email\":\"\"}],\"slot\":{\"slot_id\":\"684179245\"},\"package\":[{\"deal_id\":[\"profile_1\"]}],\"customer_calling_number\":\"8952954319\",\"billing_cust_name\":\"Rajesh QAC\",\"gender\":\"F\",\"mobile\":\"8952954319\",\"state\":\"26\",\"cityId\":\"23\",\"sub_locality\":\"vaishali nagar, jaipur\",\"latitude\":28.5100601,\"longitude\":77.0823482,\"address\":\"vaishali nagar, jaipur\",\"zipcode\":122016,\"altemail\":\"\",\"hard_copy\":0,\"vendor_booking_id\":\"TWC-974-38TKZX\",\"vendor_billing_user_id\":\"rwe18y79gxv5l6gz57yj4kod630qpm2z\",\"payment_option\":\"invoice\",\"discounted_price\":1199,\"sample_collected\":\"n\",\"role_id\":0,\"search_by\":\"customer_mobile\",\"channel_partner\":\"3\",\"channel_partner_user\":\"887\",\"booked_source\":\"api\",\"zip_search\":2,\"calling_from\":\"external_api\",\"b2bNewExistingUser\":false,\"user_id_ajax\":\"99797320352578\",\"billing_zone_id\":\"53\",\"zone_id\":\"53\",\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"billing_cust_city\":\"gurgaon\",\"billing_cust_city_id\":\"23\",\"locality\":\"614\",\"channel_type\":\"3\",\"channel_user\":\"887\"},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\"},\"calling_trace\":{\"file\":\"\\/var\\/www\\/newcrm-t25\\/application\\/controllers\\/crmuser\\/crmorder.php\",\"line\":1227,\"function\":\"addProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"nginx_request_id\":\"7cabecd57f1477ede470ea10c3ff450e\",\"timestamp\":\"2024-02-08 122622\"}","attributes":{"body":"{\"file\":\"\\/var\\/www\\/newcrm-t25\\/application\\/controllers\\/crmuser\\/crmorder.php\",\"line\":1227,\"function\":\"addProcessBookingLog\",\"class\":\"Enterprise_logger\"}},\"amzn_trace_id\":\"Root=1-65c47b16-4efdca58197a9400223773d9\",\"nginx_request_id\":\"7cabecd57f1477ede470ea10c3ff450e\",\"timestamp\":\"2024-02-08 122622\"}","log.file.path":"/var/log/newcrm/el-t25crm.healthians.co.in-2024-02-08.log"},"severity":0,"scope_name":""}} Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).HandleEntryError Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/transformer.go:98 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ParseWith Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:140 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:112 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:98 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/json.(*Parser).Process Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/parser/json/json.go:62 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/writer.go:53 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:122 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/parser.go:98 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/regex.(*Parser).Process Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/parser/regex/regex.go:99 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/helper/writer.go:53 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/file.(*Input).emit Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/operator/input/file/file.go:52 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader.(*Reader).ReadToEnd Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/fileconsumer/internal/reader/reader.go:106 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer.(*Manager).consume.func1 Feb 09 115712 ast002.healthians.co.in otelcol-contrib[29298]: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.88.0/fileconsumer/file.go:156
r
There are no templates/sample configs since your case is unique to you Please share your latest collector config again
n
receivers: filelog: include: [ "/var/log/newcrm/el-t25crm.healthians.co.in-2024-02-*.log" ] start_at: beginning include_file_path: true include_file_name: false poll_interval: 200ms operators: - type: regex_parser regex: '^.(?P<body>{.})' output: parsedata - type: json_parser id: parsedata parse_from: attributes.body - type: move from: attributes["body"] to: body - type: time_parser parse_from: attributes.time layout: '%Y-%m-%d,%H:%M:%S %z' # layout_type: gotime processors: batch: send_batch_size: 10000 send_batch_max_size: 11000 timeout: 10s exporters: otlp/log: endpoint: http://slog001.echl.in:4317 tls: insecure: true service: pipelines: logs: receivers: [filelog] processors: [batch] exporters: [otlp/log]
Below is the sample log that i need to parse {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"Adding Post Data","post_data":{"customer":[{"customer_id":"rwe18y79gxv5l6gz57yj4kod630qpm2z__716l2gd8ev9y4mg4ur176gov6z59nwkv","customer_name":"Samarpit Sacha","relation":"others","age":25,"gender":"M","contact_number":"8888822224","email":""}],"slot":{"slot_id":"684179245"},"package":[{"deal_id":["profile_1"]}],"customer_calling_number":"8952954319","billing_cust_name":"Rajesh QAC","gender":"F","mobile":"8952954319","state":"26","cityId":"23","sub_locality":"vaishali nagar, jaipur","latitude":28.5100601,"longitude":77.0823482,"address":"vaishali nagar, jaipur","zipcode":122016,"altemail":"","hard_copy":0,"vendor_booking_id":"TWC-974-38TKZX","vendor_billing_user_id":"rwe18y79gxv5l6gz57yj4kod630qpm2z","payment_option":"invoice","discounted_price":1199,"sample_collected":"n","role_id":0,"search_by":"customer_mobile","channel_partner":"3","channel_partner_user":"887","booked_source":"api","zip_search":2,"calling_from":"external_api","b2bNewExistingUser":false,"user_id_ajax":"99797320352578","billing_zone_id":"53","zone_id":"53","amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","billing_cust_city":"gurgaon","billing_cust_city_id":"23","locality":"614","channel_type":"3","channel_user":"887"},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":1227,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","nginx_request_id":"7cabecd57f1477ede470ea10c3ff450e","timestamp":"2024-02-08 122622"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"Booking Form Validation","booking_form_validate_reponse":{"status":true,"message":"validate successfully.","resCode":"RES0001"},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":1233,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","nginx_request_id":"7cabecd57f1477ede470ea10c3ff450e","timestamp":"2024-02-08 122622"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"gross_margin","gross_margin_key":{"active":"on"},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2046,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","nginx_request_id":"7cabecd57f1477ede470ea10c3ff450e","timestamp":"2024-02-08 122622"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"user_found","user":{"user_id":"99797320352578","name":"health two","email":"xyz@healthians.com","gender":"M","dob":"1993-08-19","dob_type":"actual","age":"29","contact_number":"8952954319","relationship_status":"self","city_id":"0"},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2057,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","nginx_request_id":"7cabecd57f1477ede470ea10c3ff450e","timestamp":"2024-02-08 122622"} {"channel":"crm_process_booking_log","log_type":"DEBUG","index_name":"crm_process_booking_log","index_type":"crm_process_booking_log","error_type":0,"message":{"data":{"STEP":"slot_details","slotDetail":{"stm_id":"684179245","time":"120000","end_time":"130000","locality_id":0,"booking_id":0,"freeze_count":0,"date":"2024-02-11","user_id":0,"collection_center_id":0,"b2b_zone_id":0,"zone_id":53,"sample_type_id":1,"isavailable":1,"isactive":1,"set_name":"set1","is_peak_hours":0,"source":"crm","order_group_id":0,"old_zone_id":0,"samplecollector_id":0,"old_set_name":"","phlebo_name":"","assign_time":"0000-00-00 000000","freeze":1,"freeze_date":"2024-02-08 122622","payment":0,"locusStatus":0,"lat":"","lng":"","dropTaskStatus":0,"autoAssignStatus":0,"pickup_type":"home","service_id":0,"consultation_id":0,"channel_type":3,"channel_user":887,"vendor_user_id":"rwe18y79gxv5l6gz57yj4kod630qpm2z","created_at":"2024-02-05 125626","updated_at":"2024-02-08 122622","reference_stm_id":0,"phlebo_id":0},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9"},"calling_trace":{"file":"\/var\/www\/newcrm-t25\/application\/controllers\/crmuser\/crmorder.php","line":2112,"function":"addProcessBookingLog","class":"Enterprise_logger"}},"amzn_trace_id":"Root=1-65c47b16-4efdca58197a9400223773d9","nginx_request_id":"7cabecd57f1477ede470ea10c3ff450e","timestamp":"2024-02-08 122622"}
r
You are still using
^.*(?P<body>{.*})
in the regex parser in your filelog receiver in otel collector config. Please change that to the suggested regex
^.*?(?P<body>{.*})
and try again
The error can't go away unless you change the regex parser config as we have been discussing since yesterday