root@5773-18608:~# cat /etc/otelcol-contrib/config.yaml
extensions:
  health_check:
  pprof:
    endpoint: 0.0.0.0:1777
  zpages:
    endpoint: 0.0.0.0:55679

receivers:
  hostmetrics:
    collection_interval: 10s
    scrapers:
      cpu:
      disk:
      filesystem:
      load:
      memory:
      network:
      paging:
  filelog:
    include: [ /var/log/*log ]
    start_at: end

processors:
  batch:
  resourcedetection:
    detectors: [env, system]
  cumulativetodelta:

exporters:
  otlp:
    endpoint: 154.61.80.126:4317
    tls:
      insecure: true
  debug:
    verbosity: detailed

service:
  pipelines:
    metrics:
      receivers: [hostmetrics]
      processors: [cumulativetodelta, batch, resourcedetection]
      exporters: [otlp, debug]
    logs:
      receivers: [filelog]
      processors: [batch, resourcedetection]
      exporters: [otlp, debug]

  extensions: [health_check, pprof, zpages]

{"level":"INFO","timestamp":"2025-03-08T07:21:02.350Z","logger":"<http://go.signoz.io/pkg/http/middleware|go.signoz.io/pkg/http/middleware>","caller":"middleware/logging.go:88","msg":"::RECEIVED-REQUEST::","client.address":"172.18.0.7:51088","user_agent.original":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36","server.address":"query-service","server.port":"8080","http.request.size":127,"http.route":"/api/v1/hosts/list","http.response.status_code":500,"http.server.request.duration":64,"response.body":"{\"status\":\"error\",\"errorType\":\"internal\",\"error\":\"context canceled\"}"} {"level":"ERROR","timestamp":"2025-03-08T07:21:02.367Z","caller":"clickhouseReader/reader.go:4596","msg":"error while reading result","error":"code: 173, message: Couldn't allocate 133 bytes when parsing JSON: while executing 'FUNCTION JSONExtractString(labels : 0, 'host_name' :: 1) -> JSONExtractString(labels, 'host_name') String : 4': While executing MergeTreeSelect(pool: ReadPool, algorithm: Thread)","stacktrace":"<http://go.signoz.io/signoz/pkg/query-service/app/clickhouseReader.getPersonalisedError|go.signoz.io/signoz/pkg/query-service/app/clickhouseReader.getPersonalisedError>\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/clickhouseReader/reader.go:4596\ngo.signoz.io/signoz/pkg/query-service/app/clickhouseReader.(*ClickHouseReader).GetListResultV3\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/clickhouseReader/reader.go:4588\ngo.signoz.io/signoz/pkg/query-service/app/inframetrics.(*HostsRepo).IsSendingK8SAgentMetrics\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/inframetrics/hosts.go:359\ngo.signoz.io/signoz/pkg/query-service/app/inframetrics.(*HostsRepo).GetHostList\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/inframetrics/hosts.go:406\ngo.signoz.io/signoz/pkg/query-service/app.(*APIHandler).getHostList\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/infra.go:63\ngo.signoz.io/signoz/pkg/query-service/app.(*APIHandler).RegisterInfraMetricsRoutes.(*AuthMiddleware).ViewAccess.func3\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/auth.go:50\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Logging).Wrap-fm.(*Logging).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/logging.go:67\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Analytics).Wrap-fm.(*Analytics).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/analytics.go:39\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Timeout).Wrap-fm.(*Timeout).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/timeout.go:73\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/ee/http/middleware.(*Pat).Wrap-fm.(*Pat).Wrap.func1\n\t/home/runner/work/signoz/signoz/ee/http/middleware/pat.go:27\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Auth).Wrap-fm.(*Auth).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/auth.go:41\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngithub.com/gorilla/mux.(*Router).ServeHTTP\n\t/home/runner/go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212\ngo.signoz.io/signoz/ee/query-service/app.(*Server).createPublicServer.(*Cors).Handler.func2\n\t/home/runner/go/pkg/mod/github.com/rs/cors@v1.11.1/cors.go:289\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/ee/query-service/app.(*Server).createPublicServer.CompressHandler.CompressHandlerLevel.func3\n\t/home/runner/go/pkg/mod/github.com/gorilla/handlers@v1.5.1/compress.go:141\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\nnet/http.serverHandler.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:3142\nnet/http.(*conn).serve\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2044"} {"level":"INFO","timestamp":"2025-03-08T07:21:02.368Z","caller":"utils/time.go:17","msg":"Elapsed time","func_name":"GetListResultV3","duration":65,"query":"\n\tSELECT DISTINCT JSONExtractString(labels, 'k8s_cluster_name') as k8s_cluster_name, JSONExtractString(labels, 'k8s_node_name') as k8s_node_name\n\tFROM signoz_metrics.distributed_time_series_v4\n\tWHERE metric_name IN ('system_memory_usage','system_cpu_load_average_15m','system_cpu_time','system_cpu_time')\n\t\tAND unix_milli >= toUnixTimestamp(now() - INTERVAL 60 MINUTE) * 1000\n\t\tAND JSONExtractString(labels, 'host_name') LIKE '%-otel-agent%'\n\t\tAND fingerprint GLOBAL IN (\n\tSELECT DISTINCT fingerprint\n\tFROM signoz_metrics.distributed_samples_v4\n\tWHERE metric_name IN ('system_memory_usage','system_cpu_load_average_15m','system_cpu_time','system_cpu_time')\n\t\tAND unix_milli >= toUnixTimestamp(now() - INTERVAL 5 MINUTE) * 1000)","path":"/infrastructure-monitoring/hosts","dashboardID":"","source":"","email":"<mailto:sairaj.bhagat@factsonline.in|sairaj.bhagat@factsonline.in>","alertID":"","client":"browser","viewName":"","servicesTab":""} {"level":"ERROR","timestamp":"2025-03-08T07:21:02.421Z","caller":"clickhouseReader/reader.go:4596","msg":"error while reading result","error":"code: 173, message: Couldn't allocate 118 bytes when parsing JSON: while executing 'FUNCTION JSONExtractString(labels : 1, 'host_name' :: 2) -> JSONExtractString(labels, 'host_name') String : 4'","stacktrace":"<http://go.signoz.io/signoz/pkg/query-service/app/clickhouseReader.getPersonalisedError|go.signoz.io/signoz/pkg/query-service/app/clickhouseReader.getPersonalisedError>\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/clickhouseReader/reader.go:4596\ngo.signoz.io/signoz/pkg/query-service/app/clickhouseReader.(*ClickHouseReader).GetListResultV3\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/clickhouseReader/reader.go:4588\ngo.signoz.io/signoz/pkg/query-service/app/inframetrics.(*HostsRepo).getMetadataAttributes\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/inframetrics/hosts.go:221\ngo.signoz.io/signoz/pkg/query-service/app/inframetrics.(*HostsRepo).GetHostList\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/inframetrics/hosts.go:431\ngo.signoz.io/signoz/pkg/query-service/app.(*APIHandler).getHostList\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/infra.go:63\ngo.signoz.io/signoz/pkg/query-service/app.(*APIHandler).RegisterInfraMetricsRoutes.(*AuthMiddleware).ViewAccess.func3\n\t/home/runner/work/signoz/signoz/pkg/query-service/app/auth.go:50\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Logging).Wrap-fm.(*Logging).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/logging.go:67\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Analytics).Wrap-fm.(*Analytics).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/analytics.go:39\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Timeout).Wrap-fm.(*Timeout).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/timeout.go:73\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/ee/http/middleware.(*Pat).Wrap-fm.(*Pat).Wrap.func1\n\t/home/runner/work/signoz/signoz/ee/http/middleware/pat.go:27\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/pkg/http/middleware.(*Auth).Wrap-fm.(*Auth).Wrap.func1\n\t/home/runner/work/signoz/signoz/pkg/http/middleware/auth.go:41\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngithub.com/gorilla/mux.(*Router).ServeHTTP\n\t/home/runner/go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212\ngo.signoz.io/signoz/ee/query-service/app.(*Server).createPublicServer.(*Cors).Handler.func2\n\t/home/runner/go/pkg/mod/github.com/rs/cors@v1.11.1/cors.go:289\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\ngo.signoz.io/signoz/ee/query-service/app.(*Server).createPublicServer.CompressHandler.CompressHandlerLevel.func3\n\t/home/runner/go/pkg/mod/github.com/gorilla/handlers@v1.5.1/compress.go:141\nnet/http.HandlerFunc.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2171\nnet/http.serverHandler.ServeHTTP\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:3142\nnet/http.(*conn).serve\n\t/home/runner/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.7.linux-amd64/src/net/http/server.go:2044"} {"level":"INFO","timestamp":"2025-03-08T07:21:02.421Z","caller":"utils/time.go:17","msg":"Elapsed time","func_name":"GetListResultV3","duration":30,"alertID":"","servicesTab":"","email":"<mailto:sairaj.bhagat@factsonline.in|sairaj.bhagat@factsonline.in>","path":"/infrastructure-monitoring/hosts","dashboardID":"","query":"SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, JSONExtractString(labels, 'os_type') as os_type, fingerprint FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name IN ['system_cpu_load_average_15m'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1741413600000 AND unix_milli < 1741418462345","source":"","client":"browser","viewName":""} {"level":"INFO","timestamp":"2025-03-08T07:21:02.422Z","logger":"<http://go.signoz.io/pkg/http/middleware|go.signoz.io/pkg/http/middleware>","caller":"middleware/logging.go:88","msg":"::RECEIVED-REQUEST::","client.address":"172.18.0.7:51096","user_agent.original":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36","server.address":"query-service","server.port":"8080","http.request.size":127,"http.route":"/api/v1/hosts/list","http.response.status_code":500,"http.server.request.duration":120,"response.body":"{\"status\":\"error\",\"errorType\":\"internal\",\"error\":\"code: 173, message: Couldn't allocate 118 bytes when parsing JSON: while executing 'FUNCTION JSONExtractString(labels : 1, 'host_name' :: 2) -\\u003e JSONExtractString(labels, 'host_name') String : 4'\"}"} ^Croot@6977-21141:~#

4d7d92018d1d :) SELECT * FROM system.errors;

SELECT *
FROM system.errors

Query id: 0a6a72fb-05ca-40b6-b9f5-504e770102f3

┌─name────────────────────────────────┬─code─┬─value─┬─────last_error_time─┬─last_error_message──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─last_error_trace─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────`─────────────────────────────┬─remote─┐
│ CANNOT_PARSE_INPUT_ASSERTION_FAILED │ 27 │ 60 │ 2025-03-08 07:27:47 │ Cannot parse input: expected ',' before: 'toDateTime(\'2025-03-08 07:27:47\'), \'▒\t▒▒\\\\▒▒?▒\r▒▒▒▒▒▒Y"▒>#▒▒▒IЛLoEqCg[y͜▒▒S▒▒
▒i▒▒j"▒▒5Q4▒▒ƞ▒c▒▒▒▒lX!o~}▒▒▒▒▒Dj▒;J1▒Ӆ▒▒▒▒r1eO▒▒▒\\\\By▒▒▒Z\')' │ [209719067,119744099,210136190,314355444,312073334,311930101,311928643,312031002,311992688,311989120,311985121,210687742,140455401833993,140455400936275] │ 0 │
│ UNKNOWN_IDENTIFIER │ 47 │ 6 │ 2025-03-07 16:00:25 │ Missing columns: 'resource_attributes' while processing query: 'SELECT resource_attributes FROM signoz_metrics.distributed_time_series_v2 LIMIT 10', required columns: 'resource_attributes' │ [209719067,120610700,293950848,293975771,290311365,290271020,291050520,291076119,290164473,294660630,294647674,311541289,311639737,356145778,356149361,357167175,357160573,140455401833993,140455400936275] │ 0 │
│ UNKNOWN_TABLE │ 60 │ 4 │ 2025-03-08 07:31:34 │ Table default.system does not exist │ [209719067,119654685,278590315,282160950,282200713,291204055,290259684,291050520,291076119,290164473,294660630,294647674,311541289,311639737,356145778,356149361,357167175,357160573,140455401833993,140455400936275] │ 0 │
│ SYNTAX_ERROR │ 62 │ 1 │ 2025-03-07 15:42:21 │ Syntax error: failed at position 88 ('1024'): 1024, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_. Expected identifier │ [209719067,210074733,320258332,294651123,294647674,311541289,311639737,356145778,356149361,357167175,357160573,140455401833993,140455400936275] │ 0 │
│ FILE_DOESNT_EXIST │ 107 │ 1 │ 2025-03-07 11:26:15 │ Cannot open file /proc/sys/kernel/task_delayacct │ [209719067,209996370,210047565,210059964,210317723,356198086,210303441,356234553,210290890,119630552,140455399907459,100069678] │ 0 │
│ NO_ELEMENTS_IN_CONFIG │ 139 │ 1 │ 2025-03-07 11:26:16 │ Certificate file is not set. │ [209719067,119644739,311316428,210560168,210297754,210437195,210381508,356198086,210303441,356234553,210290890,119630552,140455399907459,100069678] │ 0 │
│ CANNOT_ALLOCATE_MEMORY │ 173 │ 262 │ 2025-03-08 07:21:02 │ Couldn't allocate 118 bytes when parsing JSON │ [209719067,120574480,129750441,129863138,265988030,265990974,265996507,282321049,314570771,243060464,311917298,312031002,311992688,311989120,312045731,210687742,140455401833993,140455400936275] │ 0 │
│ NETWORK_ERROR │ 210 │ 1013 │ 2025-03-08 07:32:57 │ Connection reset by peer, while writing to socket (127.0.0.1:8123 -> 127.0.0.1:41004) │ [209719067,212494753,212493597,311709269,311712136,119695976,311497635,311682170,356145778,356149361,357167175,357160573,140455401833993,140455400936275] │ 0 │
└─────────────────────────────────────┴──────┴───────┴─────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────┘

4d7d92018d1d :) SELECT DISTINCT(JSONExtractString(labels, 'host_name'))
FROM signoz_metrics.distributed_time_series_v2;

SELECT DISTINCT JSONExtractString(labels, 'host_name')
FROM signoz_metrics.distributed_time_series_v2

Query id: a4b8fc82-35b2-451c-b409-3031a53ce084


Elapsed: 0.036 sec.

Received exception from server (version 24.1.2):
Code: 173. DB::Exception: Received from localhost:9000. DB::Exception: Couldn't allocate 336 bytes when parsing JSON: while executing 'FUNCTION JSONExtractString(labels :: 0, 'host_name' :: 1) -> JSONExtractString(labels, 'host_name') String : 2'. (CANNOT_ALLOCATE_MEMORY)

4d7d92018d1d :)