refactor: rehome prometheus operational gauges

apps/backend/.env.example

@@ -38,6 +38,11 @@ MIN_NUM_DATASETS_FOR_CHECKS=1
 PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # Run providers refresh every 4 hours
 DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # Run data retention poll every 60 minutes
 
+# Prometheus Metrics Configuration
+# Cache TTL for wallet balance collection (in seconds)
+PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # Refresh wallet balance every 1 hour
+PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # Wait 1 minute before retry after error
+
 # Maintenance windows (UTC)
 DEALBOT_MAINTENANCE_WINDOWS_UTC=07:00,22:00
 DEALBOT_MAINTENANCE_WINDOW_MINUTES=20

apps/backend/src/app.module.ts

@@ -9,6 +9,7 @@ import { DevToolsModule } from "./dev-tools/dev-tools.module.js";
 import { JobsModule } from "./jobs/jobs.module.js";
 import { MetricsModule } from "./metrics/metrics.module.js";
 import { MetricsPrometheusModule } from "./metrics-prometheus/metrics-prometheus.module.js";
+import { ProvidersModule } from "./providers/providers.module.js";
 import { RetrievalModule } from "./retrieval/retrieval.module.js";
 
 @Module({
@@ -25,6 +26,7 @@ import { RetrievalModule } from "./retrieval/retrieval.module.js";
     RetrievalModule,
     DataSourceModule,
     MetricsModule,
+    ProvidersModule,
     ...(process.env.ENABLE_DEV_MODE === "true" ? [DevToolsModule] : []),
   ],
   controllers: [AppController],

apps/backend/src/config/app.config.ts

@@ -12,6 +12,8 @@ export const configValidationSchema = Joi.object({
   DEALBOT_METRICS_PORT: Joi.number().default(9090),
   DEALBOT_METRICS_HOST: Joi.string().default("0.0.0.0"),
   ENABLE_DEV_MODE: Joi.boolean().default(false),
+  PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: Joi.number().min(60).default(3600),
+  PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS: Joi.number().min(1).default(60),
 
   // Database
   DATABASE_HOST: Joi.string().required(),
@@ -88,6 +90,8 @@ export interface IAppConfig {
   metricsPort: number;
   metricsHost: string;
   enableDevMode: boolean;
+  prometheusWalletBalanceTtlSeconds: number;
+  prometheusWalletBalanceErrorCooldownSeconds: number;
 }
 
 export interface IDatabaseConfig {
@@ -249,6 +253,14 @@ export function loadConfig(): IConfig {
       metricsPort: Number.parseInt(process.env.DEALBOT_METRICS_PORT || "9090", 10),
       metricsHost: process.env.DEALBOT_METRICS_HOST || "0.0.0.0",
       enableDevMode: process.env.ENABLE_DEV_MODE === "true",
+      prometheusWalletBalanceTtlSeconds: Number.parseInt(
+        process.env.PROMETHEUS_WALLET_BALANCE_TTL_SECONDS || "3600",
+        10,
+      ),
+      prometheusWalletBalanceErrorCooldownSeconds: Number.parseInt(
+        process.env.PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS || "60",
+        10,
+      ),
     },
     database: {
       host: process.env.DATABASE_HOST || "localhost",

apps/backend/src/data-retention/data-retention.service.spec.ts

@@ -5,7 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 import type { IConfig } from "../config/app.config.js";
 import type { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js";
 import { StorageProvider } from "../database/entities/storage-provider.entity.js";
-import { buildCheckMetricLabels } from "../metrics/utils/check-metric-labels.js";
+import { buildCheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js";
 import type { PDPSubgraphService } from "../pdp-subgraph/pdp-subgraph.service.js";
 import type { ProviderDataSetResponse } from "../pdp-subgraph/types.js";
 import type { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js";

apps/backend/src/data-retention/data-retention.service.ts

@@ -8,7 +8,7 @@ import { toStructuredError } from "../common/logging.js";
 import { IConfig } from "../config/app.config.js";
 import { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js";
 import { StorageProvider } from "../database/entities/storage-provider.entity.js";
-import { buildCheckMetricLabels, CheckMetricLabels } from "../metrics/utils/check-metric-labels.js";
+import { buildCheckMetricLabels, CheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js";
 import { PDPSubgraphService } from "../pdp-subgraph/pdp-subgraph.service.js";
 import { type ProviderDataSetResponse } from "../pdp-subgraph/types.js";
 import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js";

apps/backend/src/deal-addons/strategies/ipni.strategy.spec.ts

@@ -5,8 +5,8 @@ import { Deal } from "../../database/entities/deal.entity.js";
 import { StorageProvider } from "../../database/entities/storage-provider.entity.js";
 import { IpniStatus, ServiceType } from "../../database/types.js";
 import { IpniVerificationService } from "../../ipni/ipni-verification.service.js";
-import { buildCheckMetricLabels } from "../../metrics/utils/check-metric-labels.js";
-import { DiscoverabilityCheckMetrics } from "../../metrics/utils/check-metrics.service.js";
+import { buildCheckMetricLabels } from "../../metrics-prometheus/check-metric-labels.js";
+import { DiscoverabilityCheckMetrics } from "../../metrics-prometheus/check-metrics.service.js";
 import { IpniAddonStrategy } from "./ipni.strategy.js";
 
 describe("IpniAddonStrategy getPieceStatus", () => {

apps/backend/src/deal-addons/strategies/ipni.strategy.ts

@@ -14,8 +14,8 @@ import type { DealMetadata, IpniMetadata } from "../../database/types.js";
 import { IpniStatus, ServiceType } from "../../database/types.js";
 import { HttpClientService } from "../../http-client/http-client.service.js";
 import { IpniVerificationService } from "../../ipni/ipni-verification.service.js";
-import { classifyFailureStatus } from "../../metrics/utils/check-metric-labels.js";
-import { DiscoverabilityCheckMetrics } from "../../metrics/utils/check-metrics.service.js";
+import { classifyFailureStatus } from "../../metrics-prometheus/check-metric-labels.js";
+import { DiscoverabilityCheckMetrics } from "../../metrics-prometheus/check-metrics.service.js";
 
 import type { IDealAddon } from "../interfaces/deal-addon.interface.js";
 import type { AddonExecutionContext, DealConfiguration, IpniPreprocessingResult, SynapseConfig } from "../types.js";

apps/backend/src/deal/deal.service.spec.ts

@@ -16,7 +16,7 @@ import {
   DataSetCreationCheckMetrics,
   DataStorageCheckMetrics,
   RetrievalCheckMetrics,
-} from "../metrics/utils/check-metrics.service.js";
+} from "../metrics-prometheus/check-metrics.service.js";
 import { RetrievalAddonsService } from "../retrieval-addons/retrieval-addons.service.js";
 import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js";
 import type { PDPProviderEx } from "../wallet-sdk/wallet-sdk.types.js";

apps/backend/src/deal/deal.service.ts

@@ -19,12 +19,12 @@ import { DealStatus, ServiceType } from "../database/types.js";
 import { DataSourceService } from "../dataSource/dataSource.service.js";
 import { DealAddonsService } from "../deal-addons/deal-addons.service.js";
 import type { DealPreprocessingResult } from "../deal-addons/types.js";
-import { buildCheckMetricLabels, classifyFailureStatus } from "../metrics/utils/check-metric-labels.js";
+import { buildCheckMetricLabels, classifyFailureStatus } from "../metrics-prometheus/check-metric-labels.js";
 import {
   DataSetCreationCheckMetrics,
   DataStorageCheckMetrics,
   RetrievalCheckMetrics,
-} from "../metrics/utils/check-metrics.service.js";
+} from "../metrics-prometheus/check-metrics.service.js";
 import { RetrievalAddonsService } from "../retrieval-addons/retrieval-addons.service.js";
 import type { RetrievalConfiguration } from "../retrieval-addons/types.js";
 import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js";

apps/backend/src/jobs/jobs.service.spec.ts

@@ -17,7 +17,11 @@ const callPrivate = <T>(target: T, key: string, ...args: unknown[]) => {
 
 describe("JobsService schedule rows", () => {
   let service: JobsService;
-  let storageProviderRepositoryMock: { find: ReturnType<typeof vi.fn>; findOne: ReturnType<typeof vi.fn> };
+  let storageProviderRepositoryMock: {
+    find: ReturnType<typeof vi.fn>;
+    findOne: ReturnType<typeof vi.fn>;
+    count: ReturnType<typeof vi.fn>;
+  };
   let jobScheduleRepositoryMock: {
     upsertSchedule: ReturnType<typeof vi.fn>;
     deleteSchedulesForInactiveProviders: ReturnType<typeof vi.fn>;
@@ -41,6 +45,8 @@ describe("JobsService schedule rows", () => {
     jobsCompletedCounter: JobsServiceDeps[15];
     jobsPausedGauge: JobsServiceDeps[16];
     jobDuration: JobsServiceDeps[17];
+    storageProvidersActive: JobsServiceDeps[18];
+    storageProvidersTested: JobsServiceDeps[19];
   };
   let baseConfigValues: Partial<IConfig>;
   let configService: JobsServiceDeps[0];
@@ -64,13 +70,16 @@ describe("JobsService schedule rows", () => {
       jobsCompletedCounter: JobsServiceDeps[15];
       jobsPausedGauge: JobsServiceDeps[16];
       jobDuration: JobsServiceDeps[17];
+      storageProvidersActive: JobsServiceDeps[18];
+      storageProvidersTested: JobsServiceDeps[19];
     }>,
   ) => JobsService;
 
   beforeEach(() => {
     storageProviderRepositoryMock = {
       find: vi.fn(),
       findOne: vi.fn(),
+      count: vi.fn(),
     };
 
     jobScheduleRepositoryMock = {
@@ -102,6 +111,8 @@ describe("JobsService schedule rows", () => {
       jobsCompletedCounter: { inc: vi.fn() } as unknown as JobsServiceDeps[15],
       jobsPausedGauge: { set: vi.fn() } as unknown as JobsServiceDeps[16],
       jobDuration: { observe: vi.fn() } as unknown as JobsServiceDeps[17],
+      storageProvidersActive: { set: vi.fn() } as unknown as JobsServiceDeps[18],
+      storageProvidersTested: { set: vi.fn() } as unknown as JobsServiceDeps[19],
     };
 
     baseConfigValues = {
@@ -154,6 +165,8 @@ describe("JobsService schedule rows", () => {
         overrides.jobsCompletedCounter ?? metricsMocks.jobsCompletedCounter,
         overrides.jobsPausedGauge ?? metricsMocks.jobsPausedGauge,
         overrides.jobDuration ?? metricsMocks.jobDuration,
+        overrides.storageProvidersActive ?? metricsMocks.storageProvidersActive,
+        overrides.storageProvidersTested ?? metricsMocks.storageProvidersTested,
       );
 
     service = buildService();
@@ -1334,4 +1347,41 @@ describe("JobsService schedule rows", () => {
     // No datasets should have been created since abort was already signaled
     expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled();
   });
+
+  it("sets active, inactive, and tested provider gauge values after refresh", async () => {
+    storageProviderRepositoryMock.count
+      .mockResolvedValueOnce(10) // totalProviders
+      .mockResolvedValueOnce(7) // activeCount
+      .mockResolvedValueOnce(7); // testedCount (useOnlyApprovedProviders=false)
+
+    const activeGauge = metricsMocks.storageProvidersActive as unknown as { set: ReturnType<typeof vi.fn> };
+    const testedGauge = metricsMocks.storageProvidersTested as unknown as { set: ReturnType<typeof vi.fn> };
+
+    await callPrivate(service, "updateStorageProviderGauges");
+
+    expect(activeGauge.set).toHaveBeenCalledWith({ status: "active" }, 7);
+    expect(activeGauge.set).toHaveBeenCalledWith({ status: "inactive" }, 3);
+    expect(testedGauge.set).toHaveBeenCalledWith(7);
+  });
+
+  it("filters tested providers by isApproved when useOnlyApprovedProviders is enabled", async () => {
+    baseConfigValues.blockchain = {
+      useOnlyApprovedProviders: true,
+      minNumDataSetsForChecks: 1,
+    } as IConfig["blockchain"];
+    service = buildService();
+
+    storageProviderRepositoryMock.count.mockResolvedValueOnce(10).mockResolvedValueOnce(7).mockResolvedValueOnce(5); // testedCount (only approved)
+
+    await callPrivate(service, "updateStorageProviderGauges");
+
+    expect(storageProviderRepositoryMock.count).toHaveBeenNthCalledWith(3, {
+      where: { isActive: true, isApproved: true },
+    });
+  });
+
+  it("catches storage provider gauge errors without rethrowing", async () => {
+    storageProviderRepositoryMock.count.mockRejectedValueOnce(new Error("db error"));
+    await expect(callPrivate(service, "updateStorageProviderGauges")).resolves.toBeUndefined();
+  });
 });

apps/backend/src/jobs/jobs.service.ts

@@ -86,6 +86,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
     private readonly jobsPausedGauge: Gauge,
     @InjectMetric("job_duration_seconds")
     private readonly jobDuration: Histogram,
+    @InjectMetric("storage_providers_active")
+    private readonly storageProvidersActive: Gauge,
+    @InjectMetric("storage_providers_tested")
+    private readonly storageProvidersTested: Gauge,
   ) {}
 
   /**
@@ -635,13 +639,37 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown {
           event: "chain_integration_disabled",
           message: "Chain integration disabled; skipping provider refresh job.",
         });
-        return "success";
+      } else {
+        await this.walletSdkService.loadProviders();
       }
-      await this.walletSdkService.loadProviders();
+      await this.updateStorageProviderGauges();
       return "success";
     });
   }
 
+  private async updateStorageProviderGauges(): Promise<void> {
+    try {
+      const totalProviders = await this.storageProviderRepository.count();
+      const activeCount = await this.storageProviderRepository.count({ where: { isActive: true } });
+      const inactiveCount = Math.max(0, totalProviders - activeCount);
+
+      this.storageProvidersActive.set({ status: "active" }, activeCount);
+      this.storageProvidersActive.set({ status: "inactive" }, inactiveCount);
+
+      const useOnlyApprovedProviders = this.configService.get("blockchain").useOnlyApprovedProviders;
+      const testedCount = await this.storageProviderRepository.count({
+        where: useOnlyApprovedProviders ? { isActive: true, isApproved: true } : { isActive: true },
+      });
+      this.storageProvidersTested.set(testedCount);
+    } catch (error) {
+      this.logger.warn({
+        event: "update_storage_provider_metrics_failed",
+        message: "Failed to update storage provider metrics",
+        error: toStructuredError(error),
+      });
+    }
+  }
+
   private async handleDataSetCreationJob(job: SpJob): Promise<void> {
     const data = job.data;
     const spAddress = data.spAddress;

apps/backend/src/metrics/utils/check-metrics.service.ts → apps/backend/src/metrics-prometheus/check-metrics.service.ts

@@ -1,8 +1,8 @@
 import { Injectable, Logger } from "@nestjs/common";
 import { InjectMetric } from "@willsoto/nestjs-prometheus";
 import type { Counter, Histogram } from "prom-client";
-import type { Deal } from "../../database/entities/deal.entity.js";
-import type { RetrievalExecutionResult } from "../../retrieval-addons/types.js";
+import type { Deal } from "../database/entities/deal.entity.js";
+import type { RetrievalExecutionResult } from "../retrieval-addons/types.js";
 import { buildCheckMetricLabels, type CheckMetricLabels } from "./check-metric-labels.js";
 
 const metricsLogger = new Logger("CheckMetrics");

apps/backend/src/metrics-prometheus/metrics-prometheus.module.ts

@@ -6,13 +6,15 @@ import {
   makeHistogramProvider,
   PrometheusModule,
 } from "@willsoto/nestjs-prometheus";
+import { WalletSdkModule } from "../wallet-sdk/wallet-sdk.module.js";
 import {
   DataSetCreationCheckMetrics,
   DataStorageCheckMetrics,
   DiscoverabilityCheckMetrics,
   RetrievalCheckMetrics,
-} from "../metrics/utils/check-metrics.service.js";
+} from "./check-metrics.service.js";
 import { MetricsPrometheusInterceptor } from "./metrics-prometheus.interceptor.js";
+import { WalletBalanceCollector } from "./wallet-balance.collector.js";
 
 const KiB = 1 << 10;
 const MiB = 1 << 20;
@@ -308,6 +310,7 @@ const metricProviders = [
 @Global()
 @Module({
   imports: [
+    WalletSdkModule,
     PrometheusModule.register({
       defaultMetrics: {
         enabled: true,
@@ -325,6 +328,7 @@ const metricProviders = [
     RetrievalCheckMetrics,
     DiscoverabilityCheckMetrics,
     DataSetCreationCheckMetrics,
+    WalletBalanceCollector,
     // HTTP metrics interceptor
     {
       provide: APP_INTERCEPTOR,
@@ -338,6 +342,7 @@ const metricProviders = [
     RetrievalCheckMetrics,
     DiscoverabilityCheckMetrics,
     DataSetCreationCheckMetrics,
+    WalletBalanceCollector,
   ],
 })
 export class MetricsPrometheusModule {}