feat: Logging and fsync delay for retention deletion

tsdb/store.go

@@ -1073,6 +1073,12 @@ func (s *Store) DeleteShard(shardID uint64) error {
 				}
 			}
 
+			const DeleteLogTrigger = 10_000
+			seriesCount := ss.Cardinality()
+			deleteStart := time.Now()
+			var deletedCount atomic.Uint64
+			var deleteLogCounter atomic.Uint64
+
 			ss.ForEach(func(id uint64) {
 				if err := sfile.DeleteSeriesID(id); err != nil {
 					sfile.Logger.Error(
@@ -1081,6 +1087,19 @@ func (s *Store) DeleteShard(shardID uint64) error {
 						zap.Uint64("shard_id", shardID),
 						zap.Error(err))
 				}
+				deleted := deletedCount.Add(1)
+				deleteLogCounter.Add(1)
+
+				if deleteLogCounter.Load() > DeleteLogTrigger {
+					deleteLogCounter.Store(0)
+					s.Logger.Info(fmt.Sprintf("DeleteShard: %d series deleted", DeleteLogTrigger),
+						zap.String("db", db),
+						zap.Uint64("shard_id", shardID),
+						zap.Uint64("deleted", deleted),
+						zap.Uint64("remaining", seriesCount-deleted),
+						zap.Uint64("total", seriesCount),
+						zap.Duration("elapsed", time.Since(deleteStart)))
+				}
 			})
 		}
 	}