[Store] Fix cache total metrics accounting on metadata removal (kvcache-ai#2520)

Author: feichai0017

mooncake-store/include/master_metric_manager.h

@@ -37,6 +37,7 @@ class MasterMetricManager {
     void inc_file_cache_nums(int64_t val = 1);
     void dec_mem_cache_nums(int64_t val = 1);
     void dec_file_cache_nums(int64_t val = 1);
+    void reset_cache_total_nums();
 
     void inc_valid_get_nums(int64_t val = 1);
     void inc_total_get_nums(int64_t val = 1);

mooncake-store/include/master_service.h

@@ -862,6 +862,8 @@ class MasterService {
             soft_pin_timeout GUARDED_BY(lock);  // optional soft pin, only
                                                 // set for vip objects
         const bool hard_pinned{false};          // immutable, set at creation
+        bool memory_cache_total_accounted{false};
+        bool disk_cache_total_accounted{false};
 
         void AddReplicas(std::vector<Replica>&& replicas) {
             replicas_.insert(replicas_.end(),
@@ -1173,6 +1175,20 @@ class MasterService {
     };
     std::array<MetadataShard, kNumShards> metadata_shards_;
 
+    static bool HasCompletedMemoryCacheReplica(const ObjectMetadata& metadata);
+    static bool HasCompletedDiskCacheReplica(const ObjectMetadata& metadata);
+    static void SyncCacheTotalAccounting(ObjectMetadata& metadata);
+    void RebuildCacheTotalAccounting();
+    static void AccountCacheTotalRemoval(ObjectMetadata& metadata);
+    std::vector<Replica> PopReplicasWithCacheTotalAccounting(
+        ObjectMetadata& metadata,
+        const std::function<bool(const Replica&)>& pred_fn);
+    std::vector<Replica> PopReplicasWithCacheTotalAccounting(
+        ObjectMetadata& metadata);
+    size_t EraseReplicasWithCacheTotalAccounting(
+        ObjectMetadata& metadata,
+        const std::function<bool(const Replica&)>& pred_fn);
+
     std::unordered_map<std::string, std::string> object_group_ids_
         GUARDED_BY(group_routing_mutex_);
     mutable std::unordered_set<std::string> groups_needing_lease_refresh_
@@ -1431,9 +1447,10 @@ class MasterService {
                 // Erase invalid memory replicas (those with unmounted
                 // segments). No client_mutex_ needed since we only check memory
                 // replicas.
-                it_->second.EraseReplicas([](const Replica& replica) {
-                    return replica.has_invalid_mem_handle();
-                });
+                service_->EraseReplicasWithCacheTotalAccounting(
+                    it_->second, [](const Replica& replica) {
+                        return replica.has_invalid_mem_handle();
+                    });
                 // If no valid replicas remain, delete the whole object.
                 if (!it_->second.IsValid()) {
                     const bool had_processing =

mooncake-store/src/master_metric_manager.cpp

@@ -836,6 +836,10 @@ void MasterMetricManager::dec_mem_cache_nums(int64_t val) {
 void MasterMetricManager::dec_file_cache_nums(int64_t val) {
     file_cache_nums_.dec(val);
 }
+void MasterMetricManager::reset_cache_total_nums() {
+    mem_cache_nums_.reset();
+    file_cache_nums_.reset();
+}
 void MasterMetricManager::inc_valid_get_nums(int64_t val) {
     valid_get_nums_.inc(val);
 }