From 840f524e6ef63f18ab16421ddeef17233a350ca8 Mon Sep 17 00:00:00 2001 From: Laura Trotta Date: Thu, 19 Jun 2025 18:46:04 +0200 Subject: [PATCH 1/3] fixing cluster stats response --- specification/_types/Stats.ts | 28 +- specification/_types/common.ts | 1 + .../cluster/stats/ClusterStatsResponse.ts | 16 +- specification/cluster/stats/types.ts | 252 +++++++++++++++--- specification/nodes/_types/Stats.ts | 9 + 5 files changed, 261 insertions(+), 45 deletions(-) diff --git a/specification/_types/Stats.ts b/specification/_types/Stats.ts index 56d8cd631f..059e31369b 100644 --- a/specification/_types/Stats.ts +++ b/specification/_types/Stats.ts @@ -109,6 +109,15 @@ export class DocStats { * Elasticsearch reclaims the disk space of deleted Lucene documents when a segment is merged. */ deleted?: long + /** + * Returns the total size in bytes of all documents in this stats. + * This value may be more reliable than store_stats.size_in_bytes in estimating the index size. + */ + total_size_in_bytes?: long + /** + * Human readable total_size_in_bytes + */ + total_size?: ByteSize } export class FielddataStats { @@ -116,6 +125,19 @@ export class FielddataStats { memory_size?: ByteSize memory_size_in_bytes: long fields?: Dictionary + global_ordinals: GlobalOrdinalsStats +} + +export class GlobalOrdinalsStats { + build_time_in_millis?: UnitMillis + build_time?: string + fields?: Dictionary +} + +export class GlobalOrdinalFieldStats { + build_time_in_millis?: UnitMillis + build_time?: string + shard_max_value_count: long } export class FieldMemoryUsage { @@ -344,6 +366,10 @@ export class SegmentsStats { * Total amount, in bytes, of memory used for stored fields across all shards assigned to selected nodes. */ stored_fields_memory_in_bytes: long + /** + * Total amount of memory used for stored fields across all shards assigned to selected nodes. + */ + stored_fields_memory?: ByteSize /** * Total amount, in bytes, of memory used for terms across all shards assigned to selected nodes. */ @@ -355,7 +381,7 @@ export class SegmentsStats { /** * Total amount of memory used for term vectors across all shards assigned to selected nodes. */ - term_vectory_memory?: ByteSize + term_vectors_memory?: ByteSize /** * Total amount, in bytes, of memory used for term vectors across all shards assigned to selected nodes. */ diff --git a/specification/_types/common.ts b/specification/_types/common.ts index 00343529a8..9812544019 100644 --- a/specification/_types/common.ts +++ b/specification/_types/common.ts @@ -215,6 +215,7 @@ export type ExpandWildcards = ExpandWildcard | ExpandWildcard[] /** * Health status of the cluster, based on the state of its primary and replica shards. + * @non_exhaustive */ export enum HealthStatus { // ES will send this enum as upper or lowercase depending on the APIs diff --git a/specification/cluster/stats/ClusterStatsResponse.ts b/specification/cluster/stats/ClusterStatsResponse.ts index 9f7f68166d..68e485c506 100644 --- a/specification/cluster/stats/ClusterStatsResponse.ts +++ b/specification/cluster/stats/ClusterStatsResponse.ts @@ -20,7 +20,13 @@ import { HealthStatus, Name, Uuid } from '@_types/common' import { long } from '@_types/Numeric' import { NodesResponseBase } from '@nodes/_types/NodesResponseBase' -import { CCSStats, ClusterIndices, ClusterNodes } from './types' +import { Dictionary } from '@spec_utils/Dictionary' +import { + CCSStats, + ClusterIndices, + ClusterNodes, + ClusterSnapshotStats +} from './types' export class StatsResponseBase extends NodesResponseBase { /** @@ -40,6 +46,14 @@ export class StatsResponseBase extends NodesResponseBase { * @doc_id cluster-nodes */ nodes: ClusterNodes + /** + * Contains stats on repository feature usage exposed in cluster stats for telemetry. + */ + repositories: Dictionary> + /** + * Contains stats cluster snapshots. + */ + snapshots: ClusterSnapshotStats /** * Health status of the cluster, based on the state of its primary and replica shards. */ diff --git a/specification/cluster/stats/types.ts b/specification/cluster/stats/types.ts index 670e291f0e..e74fc3d0eb 100644 --- a/specification/cluster/stats/types.ts +++ b/specification/cluster/stats/types.ts @@ -28,7 +28,8 @@ import { SegmentsStats, StoreStats } from '@_types/Stats' -import { Duration, DurationValue, UnitMillis } from '@_types/Time' +import { DateFormat, Duration, DurationValue, UnitMillis } from '@_types/Time' +import { IndexingPressureMemory } from '@nodes/_types/Stats' import { Dictionary } from '@spec_utils/Dictionary' export class ClusterFileSystem { @@ -37,15 +38,29 @@ export class ClusterFileSystem { * Depending on operating system or process-level restrictions, this number may be less than `nodes.fs.free_in_byes`. * This is the actual amount of free disk space the selected Elasticsearch nodes can use. */ - available_in_bytes: long + available_in_bytes?: long + /** + * Total number of bytes available to JVM in file stores across all selected nodes. + * Depending on operating system or process-level restrictions, this number may be less than `nodes.fs.free_in_byes`. + * This is the actual amount of free disk space the selected Elasticsearch nodes can use. + */ + available?: ByteSize + /** + * Total number, in bytes, of unallocated bytes in file stores across all selected nodes. + */ + free_in_bytes?: long /** * Total number of unallocated bytes in file stores across all selected nodes. */ - free_in_bytes: long + free?: ByteSize /** * Total size, in bytes, of all file stores across all selected nodes. */ - total_in_bytes: long + total_in_bytes?: long + /** + * Total size of all file stores across all selected nodes. + */ + total?: ByteSize } export class ClusterIndicesShardsIndex { @@ -89,6 +104,12 @@ export class ClusterIndices { fielddata: FielddataStats /** Contains statistics about the query cache of selected nodes. */ query_cache: QueryCacheStats + /** + * Holds a snapshot of the search usage statistics. + * Used to hold the stats for a single node that's part of a ClusterStatsNodeResponse, as well as to + * accumulate stats for the entire cluster and return them as part of the ClusterStatsResponse. + */ + search: SearchUsageStats /** Contains statistics about segments in selected nodes. */ segments: SegmentsStats /** Contains statistics about indices with shards assigned to selected nodes. */ @@ -104,6 +125,44 @@ export class ClusterIndices { * @doc_id analyzer-anatomy */ versions?: IndicesVersions[] + /** + * Contains statistics about indexed dense vector + */ + dense_vector?: DenseVectorStats + /** + * Contains statistics about indexed sparse vector + */ + sparse_vector?: SparseVectorStats +} + +export class SearchUsageStats { + total: long + queries: Dictionary + rescorers: Dictionary + sections: Dictionary + retrievers: Dictionary +} + +export class DenseVectorStats { + value_count: long + off_heap?: DenseVectorOffHeapStats +} + +export class SparseVectorStats { + value_count: long +} + +export class DenseVectorOffHeapStats { + total_size_bytes?: long + total_size?: ByteSize + total_veb_size_bytes?: long + total_veb_size?: ByteSize + total_vec_size_bytes?: long + total_vec_size?: ByteSize + total_veq_size_bytes?: long + total_veq_size?: ByteSize + total_vex_size_bytes?: long + total_vex_size?: ByteSize } export class FieldTypesMappings { @@ -114,7 +173,7 @@ export class FieldTypesMappings { /** * Contains statistics about runtime field data types used in selected nodes. */ - runtime_field_types?: RuntimeFieldTypes[] + runtime_field_types: RuntimeFieldTypes[] /** * Total number of fields in all non-system indices. */ @@ -131,6 +190,10 @@ export class FieldTypesMappings { * Total size of all mappings, in bytes, after deduplication and compression. */ total_deduplicated_mapping_size_in_bytes?: long + /** + * Source mode usage count. + */ + source_modes: Dictionary } export class FieldTypes { @@ -164,6 +227,18 @@ export class FieldTypes { * @availability serverless */ script_count?: integer + /** + * Count of mappings by index type + */ + vector_index_type_count?: Dictionary + /** + * Count of mappings by similarity + */ + vector_similarity_type_count?: Dictionary + /** + * Count of mappings by element type + */ + vector_element_type_count?: Dictionary } export class RuntimeFieldTypes { @@ -258,12 +333,22 @@ export class CharFilterTypes { * Contains statistics about tokenizer types used in selected nodes. */ tokenizer_types: FieldTypes[] + /** + * Contains statistics about synonyms types used in selected nodes. + */ + synonyms: Dictionary +} + +export class SynonymsStats { + count: integer + index_count: integer } export class IndicesVersions { index_count: integer primary_shard_count: integer - total_primary_bytes: long + total_primary_bytes?: long + total_primary_size?: ByteSize version: VersionString } @@ -276,7 +361,11 @@ export class ClusterJvm { /** * Uptime duration, in milliseconds, since JVM last started. */ - max_uptime_in_millis: DurationValue + max_uptime_in_millis?: DurationValue + /** + * Uptime duration since JVM last started. + */ + max_uptime?: Duration /** * Contains statistics about memory used by selected nodes. */ @@ -295,11 +384,19 @@ export class ClusterJvmMemory { /** * Maximum amount of memory, in bytes, available for use by the heap across all selected nodes. */ - heap_max_in_bytes: long + heap_max_in_bytes?: long + /** + * Maximum amount of memory available for use by the heap across all selected nodes. + */ + heap_max?: ByteSize /** * Memory, in bytes, currently in use by the heap across all selected nodes. */ - heap_used_in_bytes: long + heap_used_in_bytes?: long + /** + * Memory currently in use by the heap across all selected nodes. + */ + heap_used?: ByteSize } export class ClusterJvmVersion { @@ -346,24 +443,26 @@ export class ClusterNetworkTypes { } export class ClusterNodeCount { - coordinating_only: integer - data: integer - data_cold: integer - data_content: integer + total: integer + coordinating_only?: integer + data?: integer + data_cold?: integer + data_content?: integer /** * @availability stack since=7.13.0 * @availability serverless */ data_frozen?: integer - data_hot: integer - data_warm: integer - ingest: integer - master: integer - ml: integer - remote_cluster_client: integer - total: integer - transform: integer - voting_only: integer + data_hot?: integer + data_warm?: integer + index?: integer + ingest?: integer + master?: integer + ml?: integer + remote_cluster_client?: integer + search?: integer + transform?: integer + voting_only?: integer } export class ClusterNodes { @@ -523,6 +622,69 @@ export class ClusterShardMetrics { min: double } +export class ClusterSnapshotStats { + current_counts: SnapshotCurrentCounts + repositories: Dictionary +} + +export class SnapshotCurrentCounts { + /** + * Snapshots currently in progress + */ + snapshots: integer + /** + * Incomplete shard snapshots + */ + shard_snapshots: integer + /** + * Snapshots deletions in progress + */ + snapshot_deletions: integer + /** + * Sum of snapshots and snapshot_deletions + */ + concurrent_operations: integer + /** + * Cleanups in progress, not counted in concurrent_operations as they are not concurrent + */ + cleanups: integer +} + +export class PerRepositoryStats { + type: string + oldest_start_time_millis?: UnitMillis + oldest_start_time?: DateFormat + current_counts: RepositoryStatsCurrentCounts +} + +export class RepositoryStatsCurrentCounts { + snapshots: integer + clones: integer + finalizations: integer + deletions: integer + snapshot_deletions: integer + active_deletions: integer + shards: RepositoryStatsShards +} + +export class RepositoryStatsShards { + total: integer + complete: integer + incomplete: integer + states: Dictionary +} + +export enum ShardState { + INIT, + SUCCESS, + FAILED, + ABORTED, + MISSING, + WAITING, + QUEUED, + PAUSED_FOR_NODE_REMOVAL +} + export class NodePackagingType { /** * Number of selected nodes using the distribution flavor and file type. @@ -545,10 +707,22 @@ export class OperatingSystemMemoryInfo { * @availability serverless */ adjusted_total_in_bytes?: long + /** + * Total amount of memory across all selected nodes, but using the value specified using the `es.total_memory_bytes` system property instead of measured total memory for those nodes where that system property was set. + * @availability stack since=7.16.0 + * @availability serverless + */ + adjusted_total?: ByteSize + /** * Amount, in bytes, of free physical memory across all selected nodes. */ - free_in_bytes: long + free_in_bytes?: long + /** + * Amount of free physical memory across all selected nodes. + */ + free?: ByteSize + /** * Percentage of free physical memory across all selected nodes. */ @@ -556,11 +730,20 @@ export class OperatingSystemMemoryInfo { /** * Total amount, in bytes, of physical memory across all selected nodes. */ - total_in_bytes: long + total_in_bytes?: long + /** + * Total amount of physical memory across all selected nodes. + */ + total?: ByteSize + /** * Amount, in bytes, of physical memory in use across all selected nodes. */ - used_in_bytes: long + used_in_bytes?: long + /** + * Amount of physical memory in use across all selected nodes. + */ + used?: ByteSize /** * Percentage of physical memory in use across all selected nodes. */ @@ -571,23 +754,6 @@ export class IndexingPressure { memory: IndexingPressureMemory } -export class IndexingPressureMemory { - current: IndexingPressureMemorySummary - limit_in_bytes: long - total: IndexingPressureMemorySummary -} - -export class IndexingPressureMemorySummary { - all_in_bytes: long - combined_coordinating_and_primary_in_bytes: long - coordinating_in_bytes: long - coordinating_rejections?: long - primary_in_bytes: long - primary_rejections?: long - replica_in_bytes: long - replica_rejections?: long -} - export class CCSStats { /** * Contains remote cluster settings and metrics collected from them. @@ -613,7 +779,7 @@ export class RemoteClusterInfo { /** The `skip_unavailable` setting used for this remote cluster. */ skip_unavailable: boolean /** Transport compression setting used for this remote cluster. */ - transport_compress: string + 'transport.compress': string /** Health status of the cluster, based on the state of its primary and replica shards. */ status: HealthStatus /** The list of Elasticsearch versions used by the nodes on the remote cluster. */ diff --git a/specification/nodes/_types/Stats.ts b/specification/nodes/_types/Stats.ts index bb6991e62a..877c5fdb12 100644 --- a/specification/nodes/_types/Stats.ts +++ b/specification/nodes/_types/Stats.ts @@ -196,6 +196,10 @@ export interface PressureMemory { * Number of indexing requests rejected in the replica stage. */ replica_rejections?: long + primary_document_rejections?: long + limit?: ByteSize + limit_in_bytes?: long + large_operation_rejections?: long } export class Discovery { @@ -932,6 +936,11 @@ export class JvmMemoryStats { * Maximum amount of memory, in bytes, available for use by the heap. */ heap_max_in_bytes?: long + /** + * Maximum amount of memory, available for use by the heap. + */ + heap_max?: ByteSize + /** * Non-heap memory used, in bytes. */ From c0399d41ebe7ec507231b93a4d8996877c214fbc Mon Sep 17 00:00:00 2001 From: Laura Trotta Date: Tue, 24 Jun 2025 12:39:56 +0200 Subject: [PATCH 2/3] addressing review with more fixes --- specification/_types/Stats.ts | 2 -- specification/cluster/stats/ClusterStatsResponse.ts | 2 +- specification/cluster/stats/types.ts | 11 +++++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/specification/_types/Stats.ts b/specification/_types/Stats.ts index 059e31369b..683bcc62c8 100644 --- a/specification/_types/Stats.ts +++ b/specification/_types/Stats.ts @@ -328,7 +328,6 @@ export class SegmentsStats { * Total amount of memory used by all index writers across all shards assigned to selected nodes. */ index_writer_memory?: ByteSize - index_writer_max_memory_in_bytes?: long /** * Total amount, in bytes, of memory used by all index writers across all shards assigned to selected nodes. */ @@ -361,7 +360,6 @@ export class SegmentsStats { * Total amount, in bytes, of memory used for points across all shards assigned to selected nodes. */ points_memory_in_bytes: long - stored_memory?: ByteSize /** * Total amount, in bytes, of memory used for stored fields across all shards assigned to selected nodes. */ diff --git a/specification/cluster/stats/ClusterStatsResponse.ts b/specification/cluster/stats/ClusterStatsResponse.ts index 68e485c506..14f582297f 100644 --- a/specification/cluster/stats/ClusterStatsResponse.ts +++ b/specification/cluster/stats/ClusterStatsResponse.ts @@ -57,7 +57,7 @@ export class StatsResponseBase extends NodesResponseBase { /** * Health status of the cluster, based on the state of its primary and replica shards. */ - status: HealthStatus + status?: HealthStatus /** * Unix timestamp, in milliseconds, for the last time the cluster statistics were refreshed. */ diff --git a/specification/cluster/stats/types.ts b/specification/cluster/stats/types.ts index e74fc3d0eb..8ea9691634 100644 --- a/specification/cluster/stats/types.ts +++ b/specification/cluster/stats/types.ts @@ -33,6 +33,9 @@ import { IndexingPressureMemory } from '@nodes/_types/Stats' import { Dictionary } from '@spec_utils/Dictionary' export class ClusterFileSystem { + path?: string + mount?: string + type?: string /** * Total number of bytes available to JVM in file stores across all selected nodes. * Depending on operating system or process-level restrictions, this number may be less than `nodes.fs.free_in_byes`. @@ -61,6 +64,14 @@ export class ClusterFileSystem { * Total size of all file stores across all selected nodes. */ total?: ByteSize + low_watermark_free_space?: ByteSize + low_watermark_free_space_in_bytes?: long + high_watermark_free_space?: ByteSize + high_watermark_free_space_in_bytes?: long + flood_stage_free_space?: ByteSize + flood_stage_free_space_in_bytes?: long + frozen_flood_stage_free_space?: ByteSize + frozen_flood_stage_free_space_in_bytes?: long } export class ClusterIndicesShardsIndex { From 443a58596e6bf7c4507da28625827f309dd875af Mon Sep 17 00:00:00 2001 From: Laura Trotta Date: Thu, 26 Jun 2025 12:58:16 +0200 Subject: [PATCH 3/3] update healthstatus enums --- specification/_global/health_report/types.ts | 3 ++- specification/_types/common.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/specification/_global/health_report/types.ts b/specification/_global/health_report/types.ts index 847dc711a5..76f67d43d5 100644 --- a/specification/_global/health_report/types.ts +++ b/specification/_global/health_report/types.ts @@ -26,7 +26,8 @@ export enum IndicatorHealthStatus { green, yellow, red, - unknown + unknown, + unavailable } export class Indicators { diff --git a/specification/_types/common.ts b/specification/_types/common.ts index 9812544019..91b93718bd 100644 --- a/specification/_types/common.ts +++ b/specification/_types/common.ts @@ -233,7 +233,9 @@ export enum HealthStatus { * One or more primary shards are unassigned, so some data is unavailable. This can occur briefly during cluster startup as primary shards are assigned. * @aliases RED */ - red + red, + unknown, + unavailable } export enum HttpMethod {