After updating some helm-values of our helm-instal...
# supporth
hans
06/25/2024, 12:01 PMAfter updating some helm-values of our helm-installation of signoz,
we got the the following stacktrace:
hans
06/25/2024, 12:01 PM Copy code
2024.06.25 11:54:49.032989 [ 7 ] {} <Information> loadMetadata: Start synchronous loading of databases
2024.06.25 11:54:49.085496 [ 702 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): Part /var/lib/clickhouse/store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44532_44532_0/ is broken and need manual correction
2024.06.25 11:54:49.092599 [ 699 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): Part /var/lib/clickhouse/store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44493_44493_0/ is broken and need manual correction
2024.06.25 11:54:49.118734 [ 699 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): while loading part 20240625_44493_44493_0 on path store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44493_44493_0: Code: 32. DB::Exception: Attempt to read after eof. (ATTEMPT_TO_READ_AFTER_EOF), Stack trace (when copying this message, always include the lines below):
0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c800f1b in /usr/bin/clickhouse
1. DB::Exception::Exception<>(int, FormatStringHelperImpl<>) @ 0x000000000721a243 in /usr/bin/clickhouse
2. DB::throwReadAfterEOF() @ 0x000000000c873ff8 in /usr/bin/clickhouse
3. void DB::readIntTextImpl<int, void, (DB::ReadIntTextCheckOverflow)0>(int&, DB::ReadBuffer&) @ 0x00000000073fdf69 in /usr/bin/clickhouse
4. DB::IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool, bool) @ 0x00000000122d2190 in /usr/bin/clickhouse
5. DB::MergeTreeData::loadDataPart(DB::MergeTreePartInfo const&, String const&, std::shared_ptr<DB::IDisk> const&, DB::MergeTreeDataPartState, std::mutex&) @ 0x0000000012366ca0 in /usr/bin/clickhouse
6. DB::MergeTreeData::loadDataPartWithRetries(DB::MergeTreePartInfo const&, String const&, std::shared_ptr<DB::IDisk> const&, DB::MergeTreeDataPartState, std::mutex&, unsigned long, unsigned long, unsigned long) @ 0x000000001236ca42 in /usr/bin/clickhouse
7. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::MergeTreeData::loadDataPartsFromDisk(std::vector<std::shared_ptr<DB::MergeTreeData::PartLoadingTree::Node>, std::allocator<std::shared_ptr<DB::MergeTreeData::PartLoadingTree::Node>>>&)::$_0, void ()>>(std::__function::__policy_storage const*) @ 0x0000000012404078 in /usr/bin/clickhouse
8. std::__packaged_task_func<std::function<std::future<void> (std::function<void ()>&&, Priority)> DB::threadPoolCallbackRunner<void, std::function<void ()>>(ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>&, String const&)::'lambda'(std::function<void ()>&&, Priority)::operator()(std::function<void ()>&&, Priority)::'lambda'(), std::allocator<std::function<std::future<void> (std::function<void ()>&&, Priority)> DB::threadPoolCallbackRunner<void, std::function<void ()>>(ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>&, String const&)::'lambda'(std::function<void ()>&&, Priority)::operator()(std::function<void ()>&&, Priority)::'lambda'()>, void ()>::operator()() @ 0x00000000104c519c in /usr/bin/clickhouse
9. std::packaged_task<void ()>::operator()() @ 0x000000000fcc4094 in /usr/bin/clickhouse
10. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false>, void*>) @ 0x000000000c8eb0c1 in /usr/bin/clickhouse
11. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c8ee8fa in /usr/bin/clickhouse
12. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c8ed6fe in /usr/bin/clickhouse
13. ? @ 0x00007f5470f1d609
14. ? @ 0x00007f5470e42353
(version 24.1.2.5 (official build))
2024.06.25 11:54:49.168084 [ 699 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): Detaching broken part /var/lib/clickhouse/store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44493_44493_0 (size: 15.39 KiB). If it happened after update, it is likely because of backward incompatibility. You need to resolve this manually
2024.06.25 11:54:49.183835 [ 705 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): Part /var/lib/clickhouse/store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44559_44559_0/ is broken and need manual correction
2024.06.25 11:54:49.208541 [ 705 ] {} <Error> signoz_traces.signoz_index_v2 (7ac63227-a642-465f-8c96-e4d3c09981d9): while loading part 20240625_44559_44559_0 on path store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44559_44559_0: Code: 228. DB::Exception: store/7ac/7ac63227-a642-465f-8c96-e4d3c09981d9/20240625_44559_44559_0/skp_idx_idx_resourceTagsMapKeys.idx has unexpected size: 0 instead of 40. (BAD_SIZE_OF_FILE_IN_DATA_PART), Stack trace (when copying this message, always include the lines below):hans
06/25/2024, 12:01 PMAnd we did not upgrade, we only changed some helm-values (added smtp for query and alert -manager)
hans
06/26/2024, 11:00 AMThis just happened again on a different cluster..
s
Srikanth Chekuri
06/26/2024, 7:27 PMDid it cause any issues other than writing error logs
h
hans
06/26/2024, 7:35 PMyeah, it wouldnt start
hans
06/26/2024, 7:38 PMi added
touch /var/lib/clickhouse/flags/force_restore_data Copy code
Loading of outdated parts failed. Will terminate to avoid undefined behaviour due to inconsistent set of parts.hans
06/26/2024, 7:38 PMbut i found a issue mentioning that error here https://github.com/ClickHouse/ClickHouse/issues/59426
hans
06/26/2024, 7:39 PMso it seems that issue was fixed in newer version of clickhous, so i changed the image-version of the statefulset, and then it ran
hans
06/26/2024, 7:39 PMand then i changed back to original version
hans
06/26/2024, 7:39 PMand now it works
hans
06/26/2024, 7:39 PMbut quite a workaround
hans
06/26/2024, 7:42 PMBut the first issue seemed to happen because it did a unhealthy shutdown when i updated the helm-template
hans
06/26/2024, 7:47 PM(this was the (2nd) issue i got after
touch /var/lib/clickhouse/flags/force_restore_datahans
06/27/2024, 7:20 AMnow it happened again on a third cluster, without any helm upgrades
hans
06/27/2024, 7:21 AM Copy code
2024.06.27 07:18:10.379538 [ 7 ] {} <Information> TablesLoader: Parsed metadata of 9 tables in 1 databases in 0.004573004 sec
2024.06.27 07:18:10.802323 [ 665 ] {} <Error> system.metric_log (19671ca3-f6c7-4980-b8b0-fd21d14b8477): Part /var/lib/clickhouse/store/196/19671ca3-f6c7-4980-b8b0-fd21d14b8477/20240627_20588_20588_0/ is broken and need manual correction
2024.06.27 07:18:10.832973 [ 7 ] {} <Warning> Context: Table system.session_log is enabled. It's unreliable and may contain garbage. Do not use it for any kind of security monitoring.
2024.06.27 07:18:10.835923 [ 24 ] {} <Error> system.metric_log (19671ca3-f6c7-4980-b8b0-fd21d14b8477): Loading of outdated parts failed. Will terminate to avoid undefined behaviour due to inconsistent set of parts. Exception: Poco::Exception. Code: 1000, e.code() = 0, JSON Exception: error: 1: unexpected end of data, Stack trace (when copying this message, always include the lines below):
0. Poco::JSON::ParserImpl::handle() @ 0x00000000154059d5 in /usr/bin/clickhouse
1. Poco::JSON::ParserImpl::handle(String const&) @ 0x000000001540516e in /usr/bin/clickhouse
2. Poco::JSON::ParserImpl::parseImpl(String const&) @ 0x0000000015405b42 in /usr/bin/clickhouse
3. DB::SerializationInfoByName::readJSON(DB::NamesAndTypesList const&, DB::SerializationInfo::Settings const&, DB::ReadBuffer&) @ 0x00000000108cc93c in /usr/bin/clickhouse
4. DB::IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool, bool) @ 0x00000000122d2029 in /usr/bin/clickhouse
5. DB::MergeTreeData::loadDataPart(DB::MergeTreePartInfo const&, String const&, std::shared_ptr<DB::IDisk> const&, DB::MergeTreeDataPartState, std::mutex&) @ 0x0000000012366ca0 in /usr/bin/clickhouse
6. DB::MergeTreeData::loadDataPartWithRetries(DB::MergeTreePartInfo const&, String const&, std::shared_ptr<DB::IDisk> const&, DB::MergeTreeDataPartState, std::mutex&, unsigned long, unsigned long, unsigned long) @ 0x000000001236ca42 in /usr/bin/clickhouse
7. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::MergeTreeData::loadOutdatedDataParts(bool)::$_0, void ()>>(std::__function::__policy_storage const*) @ 0x0000000012405659 in /usr/bin/clickhouse
8. std::__packaged_task_func<std::function<std::future<void> (std::function<void ()>&&, Priority)> DB::threadPoolCallbackRunner<void, std::function<void ()>>(ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>&, String const&)::'lambda'(std::function<void ()>&&, Priority)::operator()(std::function<void ()>&&, Priority)::'lambda'(), std::allocator<std::function<std::future<void> (std::function<void ()>&&, Priority)> DB::threadPoolCallbackRunner<void, std::function<void ()>>(ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>&, String const&)::'lambda'(std::function<void ()>&&, Priority)::operator()(std::function<void ()>&&, Priority)::'lambda'()>, void ()>::operator()() @ 0x00000000104c519c in /usr/bin/clickhouse
9. std::packaged_task<void ()>::operator()() @ 0x000000000fcc4094 in /usr/bin/clickhouse
10. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false>, void*>) @ 0x000000000c8eb0c1 in /usr/bin/clickhouse
11. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c8ee8fa in /usr/bin/clickhouse
12. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c8ed6fe in /usr/bin/clickhouse
13. ? @ 0x00007f1059fc5609
14. ? @ 0x00007f1059eea353
(version 24.1.2.5 (official build))hans
06/27/2024, 7:21 AMwell, i guess all issues look alike , and it might be this https://github.com/ClickHouse/ClickHouse/issues/59426
hans
06/27/2024, 8:25 AMSo to fix it, i need to add
touch /var/lib/clickhouse/flags/force_restore_data24.557 Views