per tenant close_idle_tsdb_timeout

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [CHANGE] Ingester: Make `close_idle_tsdb_timeout` a per-tenant limit (`-limits.close-idle-tsdb-timeout`). `-blocks-storage.tsdb.close-idle-tsdb-timeout` is deprecated and will be removed in v1.24.0. A positive `close_idle_tsdb_timeout` must now be at least `query_store_after` when `query_ingesters_within` is 0. #7813
 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828
 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873
 * [FEATURE] Storage: Add support for Alibaba Cloud OSS as a backend for blocks, ruler, and alertmanager storage. Configured via `-<prefix>.oss.*` flags with `backend: oss`. #7907

docs/blocks-storage/querier.md

@@ -2260,15 +2260,6 @@ blocks_storage:
     # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
     [flush_blocks_on_shutdown: <boolean> | default = false]
 
-    # If TSDB has not received any data for this duration, and all blocks from
-    # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-    # to positive value, this value must be greater than
-    # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-    # prematurely, which could cause partial query results. 0 or negative value
-    # disables closing of idle TSDB.
-    # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-    [close_idle_tsdb_timeout: <duration> | default = 0s]
-
     # The size of the in-memory queue used before flushing chunks to the disk.
     # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
     [head_chunks_write_queue_size: <int> | default = 0]

docs/blocks-storage/store-gateway.md

@@ -2313,15 +2313,6 @@ blocks_storage:
     # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
     [flush_blocks_on_shutdown: <boolean> | default = false]
 
-    # If TSDB has not received any data for this duration, and all blocks from
-    # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-    # to positive value, this value must be greater than
-    # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-    # prematurely, which could cause partial query results. 0 or negative value
-    # disables closing of idle TSDB.
-    # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-    [close_idle_tsdb_timeout: <duration> | default = 0s]
-
     # The size of the in-memory queue used before flushing chunks to the disk.
     # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
     [head_chunks_write_queue_size: <int> | default = 0]

docs/configuration/config-file-reference.md

@@ -2974,15 +2974,6 @@ tsdb:
   # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
   [flush_blocks_on_shutdown: <boolean> | default = false]
 
-  # If TSDB has not received any data for this duration, and all blocks from
-  # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-  # to positive value, this value must be greater than
-  # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-  # prematurely, which could cause partial query results. 0 or negative value
-  # disables closing of idle TSDB.
-  # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-  [close_idle_tsdb_timeout: <duration> | default = 0s]
-
   # The size of the in-memory queue used before flushing chunks to the disk.
   # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
   [head_chunks_write_queue_size: <int> | default = 0]
@@ -4826,7 +4817,7 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # Maximum lookback duration for querying data from ingesters. Queries for data
 # older than this will only query the long-term storage. This is a per-tenant
 # limit that can be overridden in the runtime configuration. Must be less than
-# -blocks-storage.tsdb.close-idle-tsdb-timeout when both are positive.
+# or equal to -limits.close-idle-tsdb-timeout when both are positive.
 # CLI flag: -limits.query-ingesters-within
 [query_ingesters_within: <duration> | default = 0s]
 
@@ -4842,6 +4833,15 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -limits.shuffle-sharding-ingesters-lookback-period
 [shuffle_sharding_ingesters_lookback_period: <duration> | default = 0s]
 
+# If TSDB has not received any data for this duration, and all blocks from TSDB
+# have been shipped, TSDB is closed and deleted from local disk. This is a
+# per-tenant limit that can be overridden in the runtime configuration. When
+# positive, it must be greater than or equal to -limits.query-ingesters-within
+# and -limits.query-store-after, so that a TSDB is not closed while queriers
+# still read its data only from ingesters. 0 to disable.
+# CLI flag: -limits.close-idle-tsdb-timeout
+[close_idle_tsdb_timeout: <duration> | default = 0s]
+
 # The maximum number of rows that can be fetched when querying parquet storage.
 # Each row maps to a series in a parquet file. This limit applies before
 # materializing chunks. 0 to disable.

pkg/cortex/cortex.go

@@ -315,7 +315,7 @@ func (c *Config) Validate(log log.Logger) error {
 	if err := c.LimitsConfig.Validate(c.NameValidationScheme, c.Distributor.ShardByAllLabels, c.Ingester.ActiveSeriesMetricsEnabled, c.Distributor.HATrackerConfig.UpdateTimeout, c.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil {
 		return errors.Wrap(err, "invalid limits config")
 	}
-	if err := c.LimitsConfig.ValidateQueryLimits("default", c.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil {
+	if err := c.LimitsConfig.ValidateQueryLimits("default"); err != nil {
 		return errors.Wrap(err, "invalid query routing config")
 	}
 	if err := c.ResourceMonitor.Validate(); err != nil {
@@ -390,6 +390,7 @@ func (c *Config) deprecatedLimits() []deprecatedLimit {
 		{"querier.query_store_after", "limits.query_store_after", "querier.query-store-after", "limits.query-store-after", c.Querier.DeprecatedQueryStoreAfter, &c.LimitsConfig.QueryStoreAfter, defaults.QueryStoreAfter},
 		{"querier.shuffle_sharding_ingesters_lookback_period", "limits.shuffle_sharding_ingesters_lookback_period", "querier.shuffle-sharding-ingesters-lookback-period", "limits.shuffle-sharding-ingesters-lookback-period", c.Querier.DeprecatedShuffleShardingIngestersLookbackPeriod, &c.LimitsConfig.ShuffleShardingIngestersLookbackPeriod, defaults.ShuffleShardingIngestersLookbackPeriod},
 		{"distributor.ha_tracker.ha_tracker_failover_timeout", "limits.ha_tracker_failover_timeout", "", "distributor.ha-tracker.failover-timeout", c.Distributor.HATrackerConfig.DeprecatedFailoverTimeout, &c.LimitsConfig.HATrackerFailoverTimeout, defaults.HATrackerFailoverTimeout},
+		{"blocks_storage.tsdb.close_idle_tsdb_timeout", "limits.close_idle_tsdb_timeout", "blocks-storage.tsdb.close-idle-tsdb-timeout", "limits.close-idle-tsdb-timeout", c.BlocksStorage.TSDB.DeprecatedCloseIdleTSDBTimeout, &c.LimitsConfig.CloseIdleTSDBTimeout, defaults.CloseIdleTSDBTimeout},
 	}
 }
 

pkg/cortex/cortex_test.go

@@ -339,6 +339,25 @@ func TestConfigValidation_DeprecatedLimits(t *testing.T) {
 				assert.Equal(t, model.Duration(45*time.Second), cfg.LimitsConfig.HATrackerFailoverTimeout)
 			},
 		},
+		{
+			name: "old blocks storage YAML field sets the default close-idle TSDB timeout limit",
+			yaml: "blocks_storage:\n  tsdb:\n    close_idle_tsdb_timeout: 14h\n",
+			check: func(t *testing.T, cfg *Config) {
+				assert.Equal(t, model.Duration(14*time.Hour), cfg.LimitsConfig.CloseIdleTSDBTimeout)
+			},
+		},
+		{
+			name: "old blocks storage CLI flag sets the default close-idle TSDB timeout limit",
+			args: []string{"-blocks-storage.tsdb.close-idle-tsdb-timeout=14h"},
+			check: func(t *testing.T, cfg *Config) {
+				assert.Equal(t, model.Duration(14*time.Hour), cfg.LimitsConfig.CloseIdleTSDBTimeout)
+			},
+		},
+		{
+			name:          "old and new close-idle TSDB timeout set to different values",
+			yaml:          "blocks_storage:\n  tsdb:\n    close_idle_tsdb_timeout: 14h\nlimits:\n  close_idle_tsdb_timeout: 15h\n",
+			expectedError: "are both set: remove blocks_storage.tsdb.close_idle_tsdb_timeout",
+		},
 		{
 			name: "old and new set to the same value",
 			yaml: "querier:\n  query_store_after: 12h\nlimits:\n  query_store_after: 12h\n",

pkg/cortex/runtime_config.go

@@ -80,7 +80,7 @@ func (l runtimeConfigLoader) load(r io.Reader) (any, error) {
 					if err := ul.Validate(l.cfg.NameValidationScheme, l.cfg.Distributor.ShardByAllLabels, l.cfg.Ingester.ActiveSeriesMetricsEnabled, l.cfg.Distributor.HATrackerConfig.UpdateTimeout, l.cfg.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil {
 						return nil, err
 					}
-					if err := ul.ValidateQueryLimits(userID, l.cfg.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil {
+					if err := ul.ValidateQueryLimits(userID); err != nil {
 						return nil, err
 					}
 				}

pkg/distributor/distributor_test.go

@@ -4939,9 +4939,9 @@ func TestDistributor_ShuffleShardingIngestersLookbackPeriod_Validation(t *testin
 			flagext.DefaultValues(limits)
 			limits.QueryStoreAfter = model.Duration(testData.queryStoreAfter)
 			limits.ShuffleShardingIngestersLookbackPeriod = model.Duration(testData.shuffleShardingIngestersLookbackPeriod)
+			limits.CloseIdleTSDBTimeout = model.Duration(13 * time.Hour)
 
-			// ValidateQueryLimits requires userID and closeIdleTSDBTimeout
-			err := limits.ValidateQueryLimits("test-user", 13*time.Hour)
+			err := limits.ValidateQueryLimits("test-user")
 
 			if testData.shouldBeValid {
 				assert.NoError(t, err, testData.description)

pkg/ingester/ingester.go

@@ -1023,7 +1023,9 @@ func (i *Ingester) starting(ctx context.Context) error {
 		servs = append(servs, shippingService)
 	}
 
-	if i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout > 0 {
+	// The close-idle TSDB timeout is now a per-tenant limit, so the timer always runs;
+	// closeAndDeleteIdleUserTSDBs decides per tenant whether any TSDB is actually closed.
+	{
 		interval := i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval
 		if interval == 0 {
 			interval = cortex_tsdb.DefaultCloseIdleTSDBInterval
@@ -3554,7 +3556,19 @@ func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error {
 			return nil
 		}
 
-		result := i.closeAndDeleteUserTSDBIfIdle(userID)
+		// A non-positive close-idle timeout, or an invalid one, disables idle-based closing for
+		// this tenant, so we skip the idle check entirely (and don't record an idleTsdbChecks
+		// result, since no check is performed). A tenant marked for deletion is the one exception:
+		// its local TSDB must still be closed and deleted regardless of the timeout.
+		timeout := i.closeIdleTSDBTimeout(userID)
+		if timeout <= 0 {
+			userDB, err := i.getTSDB(userID)
+			if err != nil || userDB == nil || !userDB.deletionMarkFound.Load() {
+				continue
+			}
+		}
+
+		result := i.closeAndDeleteUserTSDBIfIdle(userID, timeout)
 
 		i.TSDBState.idleTsdbChecks.WithLabelValues(string(result)).Inc()
 	}
@@ -3577,14 +3591,41 @@ func (i *Ingester) expirePostingsCache(ctx context.Context) error {
 	return nil
 }
 
-func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckResult {
+// closeIdleTSDBTimeout returns the close-idle TSDB timeout for the tenant, or 0 (disabled) when it is
+// shorter than the period in which queriers read the tenant's data only from ingesters. The runtime
+// config is not validated on ingesters, so a per-tenant override could otherwise close a TSDB that
+// queriers still read from.
+func (i *Ingester) closeIdleTSDBTimeout(userID string) time.Duration {
+	timeout := i.limits.CloseIdleTSDBTimeout(userID)
+	if timeout <= 0 {
+		return timeout
+	}
+
+	// With query_ingesters_within set, queriers read from ingesters up to that age; with it
+	// disabled, they read data younger than query_store_after only from ingesters.
+	minTimeout, minTimeoutName := i.limits.QueryIngestersWithin(userID), "query_ingesters_within"
+	if minTimeout <= 0 {
+		minTimeout, minTimeoutName = i.limits.QueryStoreAfter(userID), "query_store_after"
+	}
+
+	if timeout < minTimeout {
+		level.Error(i.logger).Log("msg", "invalid close_idle_tsdb_timeout, closing idle TSDB is disabled for the tenant", "user", userID, "err", fmt.Sprintf("close_idle_tsdb_timeout (%s) must be greater than or equal to %s (%s)", timeout, minTimeoutName, minTimeout))
+		return 0
+	}
+	return timeout
+}
+
+// closeAndDeleteUserTSDBIfIdle closes and deletes the tenant's TSDB if it has been idle for longer than
+// closeIdleTSDBTimeout, or if the tenant is marked for deletion. Callers must not pass a non-positive
+// timeout for a tenant whose TSDB should stay open: a TSDB counts as idle immediately with such a timeout.
+func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string, closeIdleTSDBTimeout time.Duration) tsdbCloseCheckResult {
 	userDB, err := i.getTSDB(userID)
 	if err != nil || userDB == nil || userDB.shipper == nil {
 		// We will not delete local data when not using shipping to storage.
 		return tsdbShippingDisabled
 	}
 
-	if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() {
+	if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() {
 		return result
 	}
 
@@ -3602,7 +3643,7 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes
 
 	// Verify again, things may have changed during the checks and pushes.
 	tenantDeleted := false
-	if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() {
+	if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() {
 		// This will also change TSDB state back to active (via defer above).
 		return result
 	} else if result == tsdbTenantMarkedForDeletion {