Clickhouse is unable to fix the broken parts automatically. The pod crashes with the following error. How to tweak

max_suspicious_broken_parts

in k8s ?

2025.01.20 09:03:51.109538 [ 7 ] {} <Information> Application: Shutting down storages.
2025.01.20 09:03:51.805268 [ 7 ] {} <Information> Context: Shutdown disk default
2025.01.20 09:03:51.842346 [ 7 ] {} <Information> Application: Closed all listening sockets.
2025.01.20 09:03:51.842384 [ 7 ] {} <Information> Application: Closed connections to servers for tables.
2025.01.20 09:03:51.842976 [ 7 ] {} <Information> Application: Waiting for background threads
2025.01.20 09:03:51.901059 [ 7 ] {} <Information> Application: Background threads finished in 58 ms
2025.01.20 09:03:51.907256 [ 7 ] {} <Error> Application: Code: 695. DB::Exception: Load job 'load table signoz_metrics.time_series_v4_6hrs' failed: Code: 231. DB::Exception: Suspiciously many (595 parts, 0.00 B in total) broken parts to remove while maximum allowed broken parts count is 100. You can change the maximum value with merge tree setting 'max_suspicious_broken_parts' in <merge_tree> configuration section or in table settings in .sql file (don't forget to return setting back to default value): Cannot attach table `signoz_metrics`.`time_series_v4_6hrs` from metadata file /var/lib/clickhouse/store/5d3/5d37a8a4-5b90-4749-b6c9-d3f761f69cf5/time_series_v4_6hrs.sql from query ATTACH TABLE signoz_metrics.time_series_v4_6hrs UUID '058d4b5e-e752-4919-8f51-8607d3433115' (`env` LowCardinality(String) DEFAULT 'default', `temporality` LowCardinality(String) DEFAULT 'Unspecified', `metric_name` LowCardinality(String), `description` LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), `unit` LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), `type` LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), `is_monotonic` Bool DEFAULT false CODEC(ZSTD(1)), `fingerprint` UInt64 CODEC(Delta(8), ZSTD(1)), `unix_milli` Int64 CODEC(Delta(8), ZSTD(1)), `labels` String CODEC(ZSTD(5)), INDEX idx_labels labels TYPE ngrambf_v1(4, 1024, 3, 0) GRANULARITY 1) ENGINE = ReplacingMergeTree PARTITION BY toDate(unix_milli / 1000) ORDER BY (env, temporality, metric_name, fingerprint, unix_milli) TTL toDateTime(unix_milli / 1000) + toIntervalSecond(2592000) SETTINGS ttl_only_drop_parts = 1, index_granularity = 8192. (TOO_MANY_UNEXPECTED_DATA_PARTS), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c800f1b in /usr/bin/clickhouse
1. DB::Exception::Exception<unsigned long&, String, DB::SettingFieldNumber<unsigned long> const&>(int, FormatStringHelperImpl<std::type_identity<unsigned long&>::type, std::type_identity<String>::type, std::type_identity<DB::SettingFieldNumber<unsigned long> const&>::type>, unsigned long&, String&&, DB::SettingFieldNumber<unsigned long> const&) @ 0x000000001237757c in /usr/bin/clickhouse
2. DB::MergeTreeData::loadDataParts(bool, std::optional<std::unordered_set<String, std::hash<String>, std::equal_to<String>, std::allocator<String>>>) @ 0x0000000012374e0e in /usr/bin/clickhouse
3. DB::StorageMergeTree::StorageMergeTree(DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, bool, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool) @ 0x00000000126e18c3 in /usr/bin/clickhouse
4. DB::create(DB::StorageFactory::Arguments const&) @ 0x00000000126dd66d in /usr/bin/clickhouse
5. DB::StorageFactory::get(DB::ASTCreateQuery const&, String const&, std::shared_ptr<DB::Context>, std::shared_ptr<DB::Context>, DB::ColumnsDescription const&, DB::ConstraintsDescription const&, bool) const @ 0x0000000011d6cf7b in /usr/bin/clickhouse
6. DB::createTableFromAST(DB::ASTCreateQuery, String const&, String const&, std::shared_ptr<DB::Context>, bool) @ 0x0000000010938a13 in /usr/bin/clickhouse
7. DB::DatabaseOrdinary::loadTableFromMetadata(std::shared_ptr<DB::Context>, String const&, DB::QualifiedTableName const&, std::shared_ptr<DB::IAST> const&, DB::LoadingStrictnessLevel) @ 0x0000000010959b1c in /usr/bin/clickhouse
8. void std::__function::__policy_invoker<void (DB::AsyncLoader&, std::shared_ptr<DB::LoadJob> const&)>::__call_impl<std::__function::__default_alloc_func<DB::DatabaseOrdinary::loadTableFromMetadataAsync(DB::AsyncLoader&, std::unordered_set<std::shared_ptr<DB::LoadJob>, std::hash<std::shared_ptr<DB::LoadJob>>, std::equal_to<std::shared_ptr<DB::LoadJob>>, std::allocator<std::shared_ptr<DB::LoadJob>>>, std::shared_ptr<DB::Context>, String const&, DB::QualifiedTableName const&, std::shared_ptr<DB::IAST> const&, DB::LoadingStrictnessLevel)::$_0, void (DB::AsyncLoader&, std::shared_ptr<DB::LoadJob> const&)>>(std::__function::__policy_storage const*, DB::AsyncLoader&, std::shared_ptr<DB::LoadJob> const&) @ 0x0000000010962433 in /usr/bin/clickhouse
9. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::AsyncLoader::spawn(DB::AsyncLoader::Pool&, std::unique_lock<std::mutex>&)::$_0, void ()>>(std::__function::__policy_storage const*) @ 0x000000000c9652c6 in /usr/bin/clickhouse
10. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false>, void*>) @ 0x000000000c8eb0c1 in /usr/bin/clickhouse
11. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c8ee8fa in /usr/bin/clickhouse
12. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c8ed6fe in /usr/bin/clickhouse
13. ? @ 0x00007f3485388609
14. ? @ 0x00007f34852ad353
(version 24.1.2.5 (official build)). (ASYNC_LOAD_FAILED), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c800f1b in /usr/bin/clickhouse
1. DB::Exception::Exception<String const&, String>(int, FormatStringHelperImpl<std::type_identity<String const&>::type, std::type_identity<String>::type>, String const&, String&&) @ 0x0000000007c0c81d in /usr/bin/clickhouse
2. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::AsyncLoader::spawn(DB::AsyncLoader::Pool&, std::unique_lock<std::mutex>&)::$_0, void ()>>(std::__function::__policy_storage const*) @ 0x000000000c965729 in /usr/bin/clickhouse
3. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false>, void*>) @ 0x000000000c8eb0c1 in /usr/bin/clickhouse
4. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c8ee8fa in /usr/bin/clickhouse
5. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c8ed6fe in /usr/bin/clickhouse
6. ? @ 0x00007f3485388609
7. ? @ 0x00007f34852ad353
(version 24.1.2.5 (official build))
2025.01.20 09:03:51.907356 [ 7 ] {} <Information> Application: shutting down
2025.01.20 09:03:51.907975 [ 8 ] {} <Information> BaseDaemon: Stop SignalListener thread
2025.01.20 09:03:51.929110 [ 1 ] {} <Information> Application: Child process exited normally with code 183.