diff --git a/CHANGELOG.md b/CHANGELOG.md index bac8ec047b5..dce8a4f2a40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,9 +47,10 @@ * [ENHANCEMENT] Compactor: Optimize cleaner run time. #6815 * [ENHANCEMENT] Parquet Storage: Allow percentage based dynamic shard size for Parquet Converter. #6817 * [ENHANCEMENT] Query Frontend: Enhance the performance of the JSON codec. #6816 +* [ENHANCEMENT] Compactor: Emit partition metrics separate from cleaner job. #6827 * [ENHANCEMENT] Metadata Cache: Support inmemory and multi level cache backend. #6829 * [ENHANCEMENT] Store Gateway: Allow to ignore syncing blocks older than certain time using `ignore_blocks_before`. #6830 -* [ENHANCEMENT] Compactor: Emit partition metrics separate from cleaner job. #6827 +* [ENHANCEMENT] Distributor: Add native histograms max sample size bytes limit validation. #6834 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517 * [BUGFIX] Ingester: Fix labelset data race condition. #6573 * [BUGFIX] Compactor: Cleaner should not put deletion marker for blocks with no-compact marker. #6576 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index d782f377b20..169be330687 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3542,6 +3542,10 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -validation.max-labels-size-bytes [max_labels_size_bytes: | default = 0] +# Maximum size in bytes of a native histogram sample. 0 to disable the limit. +# CLI flag: -validation.max-native-histogram-sample-size-bytes +[max_native_histogram_sample_size_bytes: | default = 0] + # Maximum length accepted for metric metadata. Metadata refers to Metric Name, # HELP and UNIT. # CLI flag: -validation.max-metadata-length diff --git a/pkg/util/validation/errors.go b/pkg/util/validation/errors.go index 5e6ba044cda..11ade9c93a5 100644 --- a/pkg/util/validation/errors.go +++ b/pkg/util/validation/errors.go @@ -262,6 +262,26 @@ func (e *nativeHistogramSchemaInvalidError) Error() string { return fmt.Sprintf("invalid native histogram schema %d for metric: %.200q. supported schema from %d to %d", e.receivedSchema, formatLabelSet(e.series), histogram.ExponentialSchemaMin, histogram.ExponentialSchemaMax) } +// nativeHistogramSampleSizeBytesExceededError is a ValidationError implementation for samples with native histogram +// exceeding the sample size bytes limit +type nativeHistogramSampleSizeBytesExceededError struct { + nhSampleSizeBytes int + series []cortexpb.LabelAdapter + limit int +} + +func newNativeHistogramSampleSizeBytesExceededError(series []cortexpb.LabelAdapter, nhSampleSizeBytes int, limit int) ValidationError { + return &nativeHistogramSampleSizeBytesExceededError{ + nhSampleSizeBytes: nhSampleSizeBytes, + series: series, + limit: limit, + } +} + +func (e *nativeHistogramSampleSizeBytesExceededError) Error() string { + return fmt.Sprintf("native histogram sample size bytes exceeded for metric (actual: %d, limit: %d) metric: %.200q", e.nhSampleSizeBytes, e.limit, formatLabelSet(e.series)) +} + // formatLabelSet formats label adapters as a metric name with labels, while preserving // label order, and keeping duplicates. If there are multiple "__name__" labels, only // first one is used as metric name, other ones will be included as regular labels. diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index ec1694c01a1..6288956e496 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -138,6 +138,7 @@ type Limits struct { MaxLabelValueLength int `yaml:"max_label_value_length" json:"max_label_value_length"` MaxLabelNamesPerSeries int `yaml:"max_label_names_per_series" json:"max_label_names_per_series"` MaxLabelsSizeBytes int `yaml:"max_labels_size_bytes" json:"max_labels_size_bytes"` + MaxNativeHistogramSampleSizeBytes int `yaml:"max_native_histogram_sample_size_bytes" json:"max_native_histogram_sample_size_bytes"` MaxMetadataLength int `yaml:"max_metadata_length" json:"max_metadata_length"` RejectOldSamples bool `yaml:"reject_old_samples" json:"reject_old_samples"` RejectOldSamplesMaxAge model.Duration `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"` @@ -257,6 +258,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLabelValueLength, "validation.max-length-label-value", 2048, "Maximum length accepted for label value. This setting also applies to the metric name") f.IntVar(&l.MaxLabelNamesPerSeries, "validation.max-label-names-per-series", 30, "Maximum number of label names per series.") f.IntVar(&l.MaxLabelsSizeBytes, "validation.max-labels-size-bytes", 0, "Maximum combined size in bytes of all labels and label values accepted for a series. 0 to disable the limit.") + f.IntVar(&l.MaxNativeHistogramSampleSizeBytes, "validation.max-native-histogram-sample-size-bytes", 0, "Maximum size in bytes of a native histogram sample. 0 to disable the limit.") f.IntVar(&l.MaxMetadataLength, "validation.max-metadata-length", 1024, "Maximum length accepted for metric metadata. Metadata refers to Metric Name, HELP and UNIT.") f.BoolVar(&l.RejectOldSamples, "validation.reject-old-samples", false, "Reject old samples.") _ = l.RejectOldSamplesMaxAge.Set("14d") diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 1d339ca62be..7f6b09c231f 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -58,6 +58,7 @@ const ( // Native Histogram specific validation reasons nativeHistogramBucketCountLimitExceeded = "native_histogram_buckets_exceeded" nativeHistogramInvalidSchema = "native_histogram_invalid_schema" + nativeHistogramSampleSizeBytesExceeded = "native_histogram_sample_size_bytes_exceeded" // RateLimited is one of the values for the reason to discard samples. // Declared here to avoid duplication in ingester and distributor. @@ -340,6 +341,12 @@ func ValidateMetadata(validateMetrics *ValidateMetrics, cfg *Limits, userID stri func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, histogramSample cortexpb.Histogram) (cortexpb.Histogram, error) { + // sample size validation for native histogram + if limits.MaxNativeHistogramSampleSizeBytes > 0 && histogramSample.Size() > limits.MaxNativeHistogramSampleSizeBytes { + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSampleSizeBytesExceeded, userID).Inc() + return cortexpb.Histogram{}, newNativeHistogramSampleSizeBytesExceededError(ls, histogramSample.Size(), limits.MaxNativeHistogramSampleSizeBytes) + } + // schema validation for native histogram if histogramSample.Schema < histogram.ExponentialSchemaMin || histogramSample.Schema > histogram.ExponentialSchemaMax { validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalidSchema, userID).Inc() diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 55c8e04aebf..11a1b2fc24c 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -349,15 +349,17 @@ func TestValidateNativeHistogram(t *testing.T) { belowMinRangeSchemaHistogram.Schema = -5 exceedMaxRangeSchemaFloatHistogram := tsdbutil.GenerateTestFloatHistogram(0) exceedMaxRangeSchemaFloatHistogram.Schema = 20 + exceedMaxSampleSizeBytesLimitFloatHistogram := tsdbutil.GenerateTestFloatHistogram(100) for _, tc := range []struct { - name string - bucketLimit int - resolutionReduced bool - histogram cortexpb.Histogram - expectedHistogram cortexpb.Histogram - expectedErr error - discardedSampleLabelValue string + name string + bucketLimit int + resolutionReduced bool + histogram cortexpb.Histogram + expectedHistogram cortexpb.Histogram + expectedErr error + discardedSampleLabelValue string + maxNativeHistogramSampleSizeBytesLimit int }{ { name: "no limit, histogram", @@ -455,12 +457,20 @@ func TestValidateNativeHistogram(t *testing.T) { expectedErr: newNativeHistogramSchemaInvalidError(lbls, int(exceedMaxRangeSchemaFloatHistogram.Schema)), discardedSampleLabelValue: nativeHistogramInvalidSchema, }, + { + name: "exceed max sample size bytes limit", + histogram: cortexpb.FloatHistogramToHistogramProto(0, exceedMaxSampleSizeBytesLimitFloatHistogram.Copy()), + expectedErr: newNativeHistogramSampleSizeBytesExceededError(lbls, 126, 100), + discardedSampleLabelValue: nativeHistogramSampleSizeBytesExceeded, + maxNativeHistogramSampleSizeBytesLimit: 100, + }, } { t.Run(tc.name, func(t *testing.T) { reg := prometheus.NewRegistry() validateMetrics := NewValidateMetrics(reg) limits := new(Limits) limits.MaxNativeHistogramBuckets = tc.bucketLimit + limits.MaxNativeHistogramSampleSizeBytes = tc.maxNativeHistogramSampleSizeBytesLimit actualHistogram, actualErr := ValidateNativeHistogram(validateMetrics, limits, userID, lbls, tc.histogram) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr, actualErr)