Vijay John Stephen
07/23/2025, 11:50 AM{"level":"info","ts":"2025-07-23T11:28:04.770Z","caller":"healthcheck/handler.go:132","msg":"Health Check state change","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"otelcol.component.id":"health_check","otelcol.component.kind":"extension","status":"ready"}
{"level":"info","ts":"2025-07-23T11:28:04.770Z","caller":"service@v0.128.0/service.go:282","msg":"Everything is ready. Begin running and processing data.","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"}}
{"level":"warn","ts":"2025-07-23T11:28:04.770Z","caller":"grpc@v1.72.2/clientconn.go:1405","msg":"[core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: \"signoz-otel-collector:4317\", ServerName: \"signoz-otel-collector:4317\", BalancerAttributes: {\"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>\": \"<%!p(bool=true)>\" }}. Err: connection error: desc = \"transport: Error while dialing: dial tcp 172.20.242.11:4317: connect: connection refused\"","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"grpc_log":true}
{"level":"info","timestamp":"2025-07-23T11:28:04.939Z","caller":"signozcol/collector.go:120","msg":"Collector service is running"}
{"level":"info","timestamp":"2025-07-23T11:28:04.939Z","caller":"service/service.go:73","msg":"Client started successfully"}
{"level":"info","timestamp":"2025-07-23T11:28:04.939Z","caller":"opamp/client.go:50","msg":"Ensuring collector is running","component":"opamp-server-client"}
{"level":"warn","ts":"2025-07-23T11:28:05.773Z","caller":"grpc@v1.72.2/clientconn.go:1405","msg":"[core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: \"signoz-otel-collector:4317\", ServerName: \"signoz-otel-collector:4317\", BalancerAttributes: {\"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>\": \"<%!p(bool=true)>\" }}. Err: connection error: desc = \"transport: Error while dialing: dial tcp 172.20.242.11:4317: connect: connection refused\"","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"grpc_log":true}
{"level":"warn","ts":"2025-07-23T11:28:07.468Z","caller":"grpc@v1.72.2/clientconn.go:1405","msg":"[core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: \"signoz-otel-collector:4317\", ServerName: \"signoz-otel-collector:4317\", BalancerAttributes: {\"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>\": \"<%!p(bool=true)>\" }}. Err: connection error: desc = \"transport: Error while dialing: dial tcp 172.20.242.11:4317: connect: connection refused\"","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"grpc_log":true}
{"level":"warn","ts":"2025-07-23T11:28:10.383Z","caller":"grpc@v1.72.2/clientconn.go:1405","msg":"[core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: \"signoz-otel-collector:4317\", ServerName: \"signoz-otel-collector:4317\", BalancerAttributes: {\"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>\": \"<%!p(bool=true)>\" }}. Err: connection error: desc = \"transport: Error while dialing: dial tcp 172.20.242.11:4317: connect: connection refused\"","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"grpc_log":true}
{"level":"warn","ts":"2025-07-23T11:28:13.801Z","caller":"grpc@v1.72.2/clientconn.go:1405","msg":"[core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: \"signoz-otel-collector:4317\", ServerName: \"signoz-otel-collector:4317\", BalancerAttributes: {\"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>\": \"<%!p(bool=true)>\" }}. Err: connection error: desc = \"transport: Error while dialing: dial tcp 172.20.242.11:4317: connect: connection refused\"","resource":{"service.instance.id":"f3f879cd-c4d2-44df-bc63-0684a4638537","service.name":"/signoz-otel-collector","service.version":"dev"},"grpc_log":true}
Config
otelCollector:
name: "otel-collector"
resources:
requests:
cpu: 100m
memory: 200Mi
limits:
cpu: "2"
memory: 3Gi
config:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
max_recv_msg_size_mib: 16
http:
endpoint: 0.0.0.0:4318
httpcheck:
targets:
%{~ for ep in endpoints ~}
- endpoint: ${ep.endpoint}
method: ${ep.method}
%{~ endfor ~}
collection_interval: 60s
mysql:
endpoint: <rds endpoint>
username: appuser
password: password
<deleted>
enabled: true
mysql.replica.sql_delay:
enabled: true
mysql.replica.time_behind_source:
enabled: true
awscloudwatchmetrics:
region: us-east-1
poll_interval: 5m
metrics:
named:
- <deleted>
# Collecting cloudwatch metrics
prometheus:
config:
scrape_configs:
- job_name: 'aws-cloudwatch-metrics'
scrape_timeout: 120s
scrape_interval: 300s
static_configs:
- targets: ['0.0.0.0:9106']
jaeger:
protocols:
grpc:
endpoint: 0.0.0.0:14250
thrift_http:
endpoint: 0.0.0.0:14268
httplogreceiver/heroku:
endpoint: 0.0.0.0:8081
source: heroku
httplogreceiver/json:
endpoint: 0.0.0.0:8082
source: json
processors:
batch:
send_batch_size: 50000
timeout: 1s
# Memory Limiter processor. If not set, it will be overridden with values based on k8s resource limits.
# ref: <https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md>
# memory_limiter: null
signozspanmetrics/delta:
metrics_exporter: signozclickhousemetrics
latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
dimensions_cache_size: 100000
dimensions:
- name: service.namespace
default: default
- name: deployment.environment
default: default
- name: signoz.collector.id
aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
extensions:
health_check:
endpoint: 0.0.0.0:13133
zpages:
endpoint: localhost:55679
pprof:
endpoint: localhost:1777
exporters:
c <deleted>
tenant_id: $${env:TENANT_ID}
cache:
provider: in_memory
otlp/local:
endpoint: "signoz-otel-collector:4317"
tls:
insecure: true
service:
telemetry:
logs:
encoding: json
extensions: [health_check, zpages, pprof]
pipelines:
traces:
receivers: [otlp, jaeger]
processors: [signozspanmetrics/delta, batch]
exporters: [clickhousetraces, metadataexporter]
metrics:
receivers: [otlp, httpcheck]
processors: [batch]
exporters: [metadataexporter, signozclickhousemetrics]
logs:
receivers: [otlp, httplogreceiver/heroku, httplogreceiver/json]
processors: [batch]
exporters: [clickhouselogsexporter, metadataexporter]
metrics/mysql:
receivers: [mysql, prometheus]
processors: []
exporters: [otlp/local]Vijay John Stephen
07/24/2025, 10:38 AMVijay John Stephen
07/25/2025, 6:04 AM{
"level": "error",
"timestamp": "2025-07-25T06:02:33.547Z",
"caller": "opamp/server_client.go:146",
"msg": "Server returned an error response: %v",
"component": "opamp-server-client",
"": "",
"stacktrace": "<http://github.com/SigNoz/signoz-otel-collector/opamp.(*serverClient).Start.func3|github.com/SigNoz/signoz-otel-collector/opamp.(*serverClient).Start.func3>\n\t/home/runner/work/signoz-otel-collector/signoz-otel-collector/opamp/server_client.go:146\ngithub.com/open-telemetry/opamp-go/client/internal.(*receivedProcessor).processErrorResponse\n\t/home/runner/go/pkg/mod/github.com/open-telemetry/opamp-go@v0.19.0/client/internal/receivedprocessor.go:247\ngithub.com/open-telemetry/opamp-go/client/internal.(*receivedProcessor).ProcessReceivedMessage\n\t/home/runner/go/pkg/mod/github.com/open-telemetry/opamp-go@v0.19.0/client/internal/receivedprocessor.go:170\ngithub.com/open-telemetry/opamp-go/client/internal.(*wsReceiver).ReceiverLoop\n\t/home/runner/go/pkg/mod/github.com/open-telemetry/opamp-go@v0.19.0/client/internal/wsreceiver.go:94"
}