diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java
index d1fcf0fb28464..386d8a9c86943 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java
@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
CompressedExponentialHistogram result = new CompressedExponentialHistogram();
BytesRef data = histoBytes.bytes().toBytesRef();
- result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
+ result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
return result;
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java
index eb0be8bc9262d..3c57955fdf77a 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java
@@ -82,6 +82,11 @@ public double sum() {
return 0;
}
+ @Override
+ public double min() {
+ return Double.NaN;
+ }
+
@Override
public long ramBytesUsed() {
return 0;
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java
index 3abfdddcf3f71..40c5020f46436 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java
@@ -102,6 +102,13 @@ public interface ExponentialHistogram extends Accountable {
*/
double sum();
+ /**
+ * Returns minimum of all values represented by this histogram.
+ *
+ * @return the minimum, NaN for empty histograms
+ */
+ double min();
+
/**
* Represents a bucket range of an {@link ExponentialHistogram}, either the positive or the negative range.
*/
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java
index 29a28db64175d..4d9716888cd95 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java
@@ -123,7 +123,9 @@ private void mergeValuesToHistogram() {
}
valueBuffer.reset();
- valueBuffer.setSum(rawValuesSum());
+ Aggregates aggregates = rawValuesAggregates();
+ valueBuffer.setSum(aggregates.sum());
+ valueBuffer.setMin(aggregates.min());
int scale = valueBuffer.scale();
// Buckets must be provided with their indices in ascending order.
@@ -162,12 +164,17 @@ private void mergeValuesToHistogram() {
valueCount = 0;
}
- private double rawValuesSum() {
+ private Aggregates rawValuesAggregates() {
+ if (valueCount == 0) {
+ return new Aggregates(0, Double.NaN);
+ }
double sum = 0;
+ double min = Double.MAX_VALUE;
for (int i = 0; i < valueCount; i++) {
sum += rawValueBuffer[i];
+ min = Math.min(min, rawValueBuffer[i]);
}
- return sum;
+ return new Aggregates(sum, min);
}
private static long estimateBaseSize(int numBuckets) {
@@ -190,4 +197,6 @@ public void close() {
circuitBreaker.adjustBreaker(-estimateBaseSize(rawValueBuffer.length));
}
}
+
+ private record Aggregates(double sum, double min) {}
}
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java
index ba71f90bf923e..8c7dd99dd909f 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java
@@ -151,7 +151,7 @@ public void add(ExponentialHistogram toAdd) {
}
buffer.setZeroBucket(zeroBucket);
buffer.setSum(a.sum() + b.sum());
-
+ buffer.setMin(nanAwareMin(a.min(), b.min()));
// We attempt to bring everything to the scale of A.
// This might involve increasing the scale for B, which would increase its indices.
// We need to ensure that we do not exceed MAX_INDEX / MIN_INDEX in this case.
@@ -231,4 +231,14 @@ private static int putBuckets(
return overflowCount;
}
+ private static double nanAwareMin(double a, double b) {
+ if (Double.isNaN(a)) {
+ return b;
+ }
+ if (Double.isNaN(b)) {
+ return a;
+ }
+ return Math.min(a, b);
+ }
+
}
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java
index 5359b3c41da15..c499cb11be7d7 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java
@@ -21,6 +21,9 @@
package org.elasticsearch.exponentialhistogram;
+import java.util.OptionalDouble;
+import java.util.OptionalLong;
+
public class ExponentialHistogramUtils {
/**
@@ -59,4 +62,44 @@ public static double estimateSum(BucketIterator negativeBuckets, BucketIterator
}
return sum;
}
+
+ /**
+ * Estimates the minimum value of the histogram based on the populated buckets.
+ * The returned value is guaranteed to be less than or equal to the exact minimum value of the histogram values.
+ * If the histogram is empty, an empty Optional is returned.
+ *
+ * Note that this method can return +-Infinity if the histogram bucket boundaries are not representable in a double.
+ *
+ * @param zeroBucket the zero bucket of the histogram
+ * @param negativeBuckets the negative buckets of the histogram
+ * @param positiveBuckets the positive buckets of the histogram
+ * @return the estimated minimum
+ */
+ public static OptionalDouble estimateMin(
+ ZeroBucket zeroBucket,
+ ExponentialHistogram.Buckets negativeBuckets,
+ ExponentialHistogram.Buckets positiveBuckets
+ ) {
+ int scale = negativeBuckets.iterator().scale();
+ assert scale == positiveBuckets.iterator().scale();
+
+ OptionalLong negativeMaxIndex = negativeBuckets.maxBucketIndex();
+ if (negativeMaxIndex.isPresent()) {
+ return OptionalDouble.of(-ExponentialScaleUtils.getUpperBucketBoundary(negativeMaxIndex.getAsLong(), scale));
+ }
+
+ if (zeroBucket.count() > 0) {
+ if (zeroBucket.zeroThreshold() == 0.0) {
+ // avoid negative zero
+ return OptionalDouble.of(0.0);
+ }
+ return OptionalDouble.of(-zeroBucket.zeroThreshold());
+ }
+
+ BucketIterator positiveBucketsIt = positiveBuckets.iterator();
+ if (positiveBucketsIt.hasNext()) {
+ return OptionalDouble.of(ExponentialScaleUtils.getLowerBucketBoundary(positiveBucketsIt.peekIndex(), scale));
+ }
+ return OptionalDouble.empty();
+ }
}
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java
index 8f0995dca6e9c..d6f840851a282 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java
@@ -32,6 +32,7 @@ public class ExponentialHistogramXContent {
public static final String SCALE_FIELD = "scale";
public static final String SUM_FIELD = "sum";
+ public static final String MIN_FIELD = "min";
public static final String ZERO_FIELD = "zero";
public static final String ZERO_COUNT_FIELD = "count";
public static final String ZERO_THRESHOLD_FIELD = "threshold";
@@ -51,6 +52,9 @@ public static void serialize(XContentBuilder builder, ExponentialHistogram histo
builder.field(SCALE_FIELD, histogram.scale());
builder.field(SUM_FIELD, histogram.sum());
+ if (Double.isNaN(histogram.min()) == false) {
+ builder.field(MIN_FIELD, histogram.min());
+ }
double zeroThreshold = histogram.zeroBucket().zeroThreshold();
long zeroCount = histogram.zeroBucket().count();
diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java
index 29fd8f01520a2..67b478af22427 100644
--- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java
+++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java
@@ -54,6 +54,7 @@ final class FixedCapacityExponentialHistogram implements ReleasableExponentialHi
private final Buckets positiveBuckets = new Buckets(true);
private double sum;
+ private double min;
private final ExponentialHistogramCircuitBreaker circuitBreaker;
private boolean closed = false;
@@ -81,6 +82,7 @@ private FixedCapacityExponentialHistogram(int bucketCapacity, ExponentialHistogr
*/
void reset() {
sum = 0;
+ min = Double.NaN;
setZeroBucket(ZeroBucket.minimalEmpty());
resetBuckets(MAX_SCALE);
}
@@ -122,6 +124,15 @@ void setSum(double sum) {
this.sum = sum;
}
+ @Override
+ public double min() {
+ return min;
+ }
+
+ void setMin(double min) {
+ this.min = min;
+ }
+
/**
* Attempts to add a bucket to the positive or negative range of this histogram.
*
diff --git a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java
index 7800270811054..17f4f172ecace 100644
--- a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java
+++ b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java
@@ -29,6 +29,7 @@
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
import java.util.stream.IntStream;
import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -106,19 +107,24 @@ public void testEmptyZeroBucketIgnored() {
assertThat(posBuckets.hasNext(), equalTo(false));
}
- public void testSumCorrectness() {
+ public void testAggregatesCorrectness() {
double[] firstValues = randomDoubles(100).map(val -> val * 2 - 1).toArray();
double[] secondValues = randomDoubles(50).map(val -> val * 2 - 1).toArray();
double correctSum = Arrays.stream(firstValues).sum() + Arrays.stream(secondValues).sum();
+ double correctMin = DoubleStream.concat(Arrays.stream(firstValues), Arrays.stream(secondValues)).min().getAsDouble();
try (
+ // Merge some empty histograms too to test that code path
ReleasableExponentialHistogram merged = ExponentialHistogram.merge(
2,
breaker(),
+ ExponentialHistogram.empty(),
createAutoReleasedHistogram(10, firstValues),
- createAutoReleasedHistogram(20, secondValues)
+ createAutoReleasedHistogram(20, secondValues),
+ ExponentialHistogram.empty()
)
) {
assertThat(merged.sum(), closeTo(correctSum, 0.000001));
+ assertThat(merged.min(), equalTo(correctMin));
}
}
diff --git a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java
index 96d9d40a37208..ca15b66983515 100644
--- a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java
+++ b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java
@@ -21,6 +21,8 @@
package org.elasticsearch.exponentialhistogram;
+import java.util.OptionalDouble;
+
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.equalTo;
@@ -57,7 +59,7 @@ public void testRandomDataSumEstimation() {
}
}
- public void testInfinityHandling() {
+ public void testSumInfinityHandling() {
FixedCapacityExponentialHistogram morePositiveValues = createAutoReleasedHistogram(100);
morePositiveValues.resetBuckets(0);
morePositiveValues.tryAddBucket(1999, 1, false);
@@ -83,4 +85,85 @@ public void testInfinityHandling() {
);
assertThat(sum, equalTo(Double.NEGATIVE_INFINITY));
}
+
+ public void testMinimumEstimation() {
+ for (int i = 0; i < 100; i++) {
+ int positiveValueCount = randomBoolean() ? 0 : randomIntBetween(10, 10_000);
+ int negativeValueCount = randomBoolean() ? 0 : randomIntBetween(10, 10_000);
+ int zeroValueCount = randomBoolean() ? 0 : randomIntBetween(10, 100);
+ int bucketCount = randomIntBetween(2, 500);
+
+ double correctMin = Double.MAX_VALUE;
+ double zeroThreshold = Double.MAX_VALUE;
+ double[] values = new double[positiveValueCount + negativeValueCount];
+ for (int j = 0; j < values.length; j++) {
+ double absValue = Math.pow(10, randomIntBetween(1, 9)) * randomDouble();
+ if (j < positiveValueCount) {
+ values[j] = absValue;
+ } else {
+ values[j] = -absValue;
+ }
+ zeroThreshold = Math.min(zeroThreshold, absValue / 2);
+ correctMin = Math.min(correctMin, values[j]);
+ }
+ if (zeroValueCount > 0) {
+ correctMin = Math.min(correctMin, -zeroThreshold);
+ }
+
+ ExponentialHistogram histo = createAutoReleasedHistogram(bucketCount, values);
+
+ OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+ new ZeroBucket(zeroThreshold, zeroValueCount),
+ histo.negativeBuckets(),
+ histo.positiveBuckets()
+ );
+ if (correctMin == Double.MAX_VALUE) {
+ assertThat(estimatedMin.isPresent(), equalTo(false));
+ } else {
+ assertThat(estimatedMin.isPresent(), equalTo(true));
+ // If the histogram does not contain mixed sign values, we have a guaranteed relative error bound of 2^(2^-scale) - 1
+ double histogramBase = Math.pow(2, Math.pow(2, -histo.scale()));
+ double allowedError = Math.abs(correctMin * (histogramBase - 1));
+ assertThat(estimatedMin.getAsDouble(), closeTo(correctMin, allowedError));
+ }
+ }
+ }
+
+ public void testMinimumEstimationPositiveInfinityHandling() {
+ FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(100);
+ histo.resetBuckets(0);
+ histo.tryAddBucket(2000, 1, true);
+
+ OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+ ZeroBucket.minimalEmpty(),
+ histo.negativeBuckets(),
+ histo.positiveBuckets()
+ );
+ assertThat(estimate.isPresent(), equalTo(true));
+ assertThat(estimate.getAsDouble(), equalTo(Double.POSITIVE_INFINITY));
+ }
+
+ public void testMinimumEstimationNegativeInfinityHandling() {
+ FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(100);
+ histo.resetBuckets(0);
+ histo.tryAddBucket(2000, 1, false);
+
+ OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+ ZeroBucket.minimalEmpty(),
+ histo.negativeBuckets(),
+ histo.positiveBuckets()
+ );
+ assertThat(estimate.isPresent(), equalTo(true));
+ assertThat(estimate.getAsDouble(), equalTo(Double.NEGATIVE_INFINITY));
+ }
+
+ public void testMinimumEstimationSanitizedNegativeZero() {
+ OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+ ZeroBucket.minimalWithCount(42),
+ ExponentialHistogram.empty().negativeBuckets(),
+ ExponentialHistogram.empty().positiveBuckets()
+ );
+ assertThat(estimate.isPresent(), equalTo(true));
+ assertThat(estimate.getAsDouble(), equalTo(0.0));
+ }
}
diff --git a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java
index 11c226eac433a..06c9e1ad4150f 100644
--- a/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java
+++ b/libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java
@@ -41,6 +41,7 @@ public void testFullHistogram() {
histo.setZeroBucket(new ZeroBucket(0.1234, 42));
histo.resetBuckets(7);
histo.setSum(1234.56);
+ histo.setMin(-321.123);
histo.tryAddBucket(-10, 15, false);
histo.tryAddBucket(10, 5, false);
histo.tryAddBucket(-11, 10, true);
@@ -51,6 +52,7 @@ public void testFullHistogram() {
"{"
+ "\"scale\":7,"
+ "\"sum\":1234.56,"
+ + "\"min\":-321.123,"
+ "\"zero\":{\"count\":42,\"threshold\":0.1234},"
+ "\"positive\":{\"indices\":[-11,11],\"counts\":[10,20]},"
+ "\"negative\":{\"indices\":[-10,10],\"counts\":[15,5]}"
@@ -72,25 +74,28 @@ public void testOnlyZeroCount() {
histo.setZeroBucket(new ZeroBucket(0.0, 7));
histo.resetBuckets(2);
histo.setSum(1.1);
- assertThat(toJson(histo), equalTo("{\"scale\":2,\"sum\":1.1,\"zero\":{\"count\":7}}"));
+ histo.setMin(0);
+ assertThat(toJson(histo), equalTo("{\"scale\":2,\"sum\":1.1,\"min\":0.0,\"zero\":{\"count\":7}}"));
}
public void testOnlyPositiveBuckets() {
FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(10);
histo.resetBuckets(4);
histo.setSum(1.1);
+ histo.setMin(0.5);
histo.tryAddBucket(-1, 3, true);
histo.tryAddBucket(2, 5, true);
- assertThat(toJson(histo), equalTo("{\"scale\":4,\"sum\":1.1,\"positive\":{\"indices\":[-1,2],\"counts\":[3,5]}}"));
+ assertThat(toJson(histo), equalTo("{\"scale\":4,\"sum\":1.1,\"min\":0.5,\"positive\":{\"indices\":[-1,2],\"counts\":[3,5]}}"));
}
public void testOnlyNegativeBuckets() {
FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(10);
histo.resetBuckets(5);
histo.setSum(1.1);
+ histo.setMin(-0.5);
histo.tryAddBucket(-1, 4, false);
histo.tryAddBucket(2, 6, false);
- assertThat(toJson(histo), equalTo("{\"scale\":5,\"sum\":1.1,\"negative\":{\"indices\":[-1,2],\"counts\":[4,6]}}"));
+ assertThat(toJson(histo), equalTo("{\"scale\":5,\"sum\":1.1,\"min\":-0.5,\"negative\":{\"indices\":[-1,2],\"counts\":[4,6]}}"));
}
private static String toJson(ExponentialHistogram histo) {
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java
index 78d3afaecf2e4..f076ca072f323 100644
--- a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java
+++ b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java
@@ -34,6 +34,7 @@ public class CompressedExponentialHistogram implements ExponentialHistogram {
private double zeroThreshold;
private long valueCount;
private double sum;
+ private double min;
private ZeroBucket lazyZeroBucket;
private final EncodedHistogramData encodedData = new EncodedHistogramData();
@@ -59,6 +60,11 @@ public double sum() {
return sum;
}
+ @Override
+ public double min() {
+ return min;
+ }
+
@Override
public ExponentialHistogram.Buckets positiveBuckets() {
return positiveBuckets;
@@ -75,14 +81,17 @@ public ExponentialHistogram.Buckets negativeBuckets() {
* @param zeroThreshold the zeroThreshold for the histogram, which needs to be stored externally
* @param valueCount the total number of values the histogram contains, needs to be stored externally
* @param sum the total sum of the values the histogram contains, needs to be stored externally
+ * @param min the minimum of the values the histogram contains, needs to be stored externally.
+ * Must be {@link Double#NaN} if the histogram is empty.
* @param encodedHistogramData the encoded histogram bytes which previously where generated via
* {@link #writeHistogramBytes(StreamOutput, int, List, List)}.
*/
- public void reset(double zeroThreshold, long valueCount, double sum, BytesRef encodedHistogramData) throws IOException {
+ public void reset(double zeroThreshold, long valueCount, double sum, double min, BytesRef encodedHistogramData) throws IOException {
lazyZeroBucket = null;
this.zeroThreshold = zeroThreshold;
this.valueCount = valueCount;
this.sum = sum;
+ this.min = min;
encodedData.decode(encodedHistogramData);
negativeBuckets.resetCachedData();
positiveBuckets.resetCachedData();
@@ -90,7 +99,7 @@ public void reset(double zeroThreshold, long valueCount, double sum, BytesRef en
/**
* Serializes the given histogram, so that exactly the same data can be reconstructed via
- * {@link #reset(double, long, double, BytesRef)}.
+ * {@link #reset(double, long, double, double, BytesRef)}.
*
* @param output the output to write the serialized bytes to
* @param scale the scale of the histogram
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
index 30715b8411304..1d33e0c7eaa23 100644
--- a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
+++ b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
@@ -22,6 +22,7 @@
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
+import org.elasticsearch.exponentialhistogram.ZeroBucket;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
@@ -48,6 +49,7 @@
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.OptionalDouble;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -72,6 +74,8 @@
* {
* "my_histo": {
* "scale": 12,
+ * "sum": 1234,
+ * "min": -123.456,
* "zero": {
* "threshold": 0.123456,
* "count": 42
@@ -95,6 +99,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
public static final ParseField SCALE_FIELD = new ParseField(ExponentialHistogramXContent.SCALE_FIELD);
public static final ParseField SUM_FIELD = new ParseField(ExponentialHistogramXContent.SUM_FIELD);
+ public static final ParseField MIN_FIELD = new ParseField(ExponentialHistogramXContent.MIN_FIELD);
public static final ParseField ZERO_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_FIELD);
public static final ParseField ZERO_COUNT_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_COUNT_FIELD);
public static final ParseField ZERO_THRESHOLD_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_THRESHOLD_FIELD);
@@ -140,6 +145,10 @@ private static String valuesSumSubFieldName(String fullPath) {
return fullPath + "._values_sum";
}
+ private static String valuesMinSubFieldName(String fullPath) {
+ return fullPath + "._values_min";
+ }
+
static class Builder extends FieldMapper.Builder {
private final FieldMapper.Parameter> meta = FieldMapper.Parameter.metaParam();
@@ -266,6 +275,7 @@ public void parse(DocumentParserContext context) throws IOException {
}
Double sum = null;
+ Double min = null;
Integer scale = null;
ParsedZeroBucket zeroBucket = ParsedZeroBucket.DEFAULT;
List negativeBuckets = Collections.emptyList();
@@ -305,6 +315,8 @@ public void parse(DocumentParserContext context) throws IOException {
}
} else if (fieldName.equals(SUM_FIELD.getPreferredName())) {
sum = parseDoubleAllowingInfinity(subParser);
+ } else if (fieldName.equals(MIN_FIELD.getPreferredName())) {
+ min = parseDoubleAllowingInfinity(subParser);
} else if (fieldName.equals(ZERO_FIELD.getPreferredName())) {
zeroBucket = parseZeroBucket(subParser);
} else if (fieldName.equals(POSITIVE_FIELD.getPreferredName())) {
@@ -347,8 +359,8 @@ public void parse(DocumentParserContext context) throws IOException {
if (sum == null) {
sum = ExponentialHistogramUtils.estimateSum(
- IndexWithCount.asBucketIterator(scale, negativeBuckets),
- IndexWithCount.asBucketIterator(scale, positiveBuckets)
+ IndexWithCount.asBuckets(scale, negativeBuckets).iterator(),
+ IndexWithCount.asBuckets(scale, positiveBuckets).iterator()
);
} else {
if (totalValueCount == 0 && sum != 0.0) {
@@ -359,6 +371,22 @@ public void parse(DocumentParserContext context) throws IOException {
}
}
+ if (min == null) {
+ OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+ new ZeroBucket(zeroBucket.threshold(), zeroBucket.count),
+ IndexWithCount.asBuckets(scale, negativeBuckets),
+ IndexWithCount.asBuckets(scale, positiveBuckets)
+ );
+ if (estimatedMin.isPresent()) {
+ min = estimatedMin.getAsDouble();
+ }
+ } else if (totalValueCount == 0) {
+ throw new DocumentParsingException(
+ subParser.getTokenLocation(),
+ "error parsing field [" + fullPath() + "], min field must be null if the histogram is empty, but got " + min
+ );
+ }
+
BytesStreamOutput histogramBytesOutput = new BytesStreamOutput();
CompressedExponentialHistogram.writeHistogramBytes(histogramBytesOutput, scale, negativeBuckets, positiveBuckets);
BytesRef histoBytes = histogramBytesOutput.bytes().toBytesRef();
@@ -376,6 +404,13 @@ public void parse(DocumentParserContext context) throws IOException {
context.doc().add(zeroThresholdField);
context.doc().add(valuesCountField);
context.doc().add(sumField);
+ if (min != null) {
+ NumericDocValuesField minField = new NumericDocValuesField(
+ valuesMinSubFieldName(fullPath()),
+ NumericUtils.doubleToSortableLong(min)
+ );
+ context.doc().add(minField);
+ }
} catch (Exception ex) {
if (ignoreMalformed.value() == false) {
@@ -629,6 +664,7 @@ private class ExponentialHistogramSyntheticFieldLoader implements CompositeSynth
private double zeroThreshold;
private long valueCount;
private double valueSum;
+ private double valueMin;
@Override
public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf)
@@ -642,6 +678,7 @@ public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafRea
NumericDocValues zeroThresholds = leafReader.getNumericDocValues(zeroThresholdSubFieldName(fullPath()));
NumericDocValues valueCounts = leafReader.getNumericDocValues(valuesCountSubFieldName(fullPath()));
NumericDocValues valueSums = leafReader.getNumericDocValues(valuesSumSubFieldName(fullPath()));
+ NumericDocValues valueMins = leafReader.getNumericDocValues(valuesMinSubFieldName(fullPath()));
assert zeroThresholds != null;
assert valueCounts != null;
assert valueSums != null;
@@ -657,6 +694,12 @@ public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafRea
zeroThreshold = NumericUtils.sortableLongToDouble(zeroThresholds.longValue());
valueCount = valueCounts.longValue();
valueSum = NumericUtils.sortableLongToDouble(valueSums.longValue());
+
+ if (valueMins != null && valueMins.advanceExact(docId)) {
+ valueMin = NumericUtils.sortableLongToDouble(valueMins.longValue());
+ } else {
+ valueMin = Double.NaN;
+ }
return true;
}
binaryValue = null;
@@ -675,7 +718,7 @@ public void write(XContentBuilder b) throws IOException {
return;
}
- histogram.reset(zeroThreshold, valueCount, valueSum, binaryValue);
+ histogram.reset(zeroThreshold, valueCount, valueSum, valueMin, binaryValue);
ExponentialHistogramXContent.serialize(b, histogram);
}
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java
index 54c23508f220a..7d9963e1e0c70 100644
--- a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java
+++ b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java
@@ -7,9 +7,11 @@
package org.elasticsearch.xpack.exponentialhistogram;
-import org.elasticsearch.exponentialhistogram.BucketIterator;
+import org.elasticsearch.exponentialhistogram.CopyableBucketIterator;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
import java.util.List;
+import java.util.OptionalLong;
/**
* An exponential histogram bucket represented by its index and associated bucket count.
@@ -17,34 +19,68 @@
* @param count the number of values in that bucket.
*/
public record IndexWithCount(long index, long count) {
- public static BucketIterator asBucketIterator(int scale, List buckets) {
- return new BucketIterator() {
- int position = 0;
+ static ExponentialHistogram.Buckets asBuckets(int scale, List bucketIndices) {
+ return new ExponentialHistogram.Buckets() {
@Override
- public boolean hasNext() {
- return position < buckets.size();
+ public CopyableBucketIterator iterator() {
+ return new Iterator(bucketIndices, scale, 0);
}
@Override
- public long peekCount() {
- return buckets.get(position).count;
+ public OptionalLong maxBucketIndex() {
+ if (bucketIndices.isEmpty()) {
+ return OptionalLong.empty();
+ }
+ return OptionalLong.of(bucketIndices.get(bucketIndices.size() - 1).index);
}
@Override
- public long peekIndex() {
- return buckets.get(position).index;
+ public long valueCount() {
+ throw new UnsupportedOperationException("not implemented");
}
+ };
+ }
- @Override
- public void advance() {
- position++;
- }
+ private static class Iterator implements CopyableBucketIterator {
+ private final List buckets;
+ private final int scale;
+ private int position;
- @Override
- public int scale() {
- return scale;
- }
- };
+ Iterator(List buckets, int scale, int position) {
+ this.buckets = buckets;
+ this.scale = scale;
+ this.position = position;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return position < buckets.size();
+ }
+
+ @Override
+ public long peekCount() {
+ return buckets.get(position).count;
+ }
+
+ @Override
+ public long peekIndex() {
+ return buckets.get(position).index;
+ }
+
+ @Override
+ public void advance() {
+ position++;
+ }
+
+ @Override
+ public int scale() {
+ return scale;
+ }
+
+ @Override
+ public CopyableBucketIterator copy() {
+ return new Iterator(buckets, scale, position);
+ }
}
}
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java
index 583e420358c22..07a7ef241f76a 100644
--- a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java
+++ b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java
@@ -9,6 +9,7 @@
import org.elasticsearch.core.Types;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
+import org.elasticsearch.exponentialhistogram.ZeroBucket;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentParsingException;
import org.elasticsearch.index.mapper.MappedFieldType;
@@ -30,6 +31,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.OptionalDouble;
import java.util.Set;
import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -129,8 +131,13 @@ private static Map createRandomHistogramValue(int maxBucketCount
Map.of("indices", negativeIndices, "counts", negativeCounts)
)
);
- if (randomBoolean() && (positiveIndices.isEmpty() == false || negativeIndices.isEmpty() == false)) {
- result.put("sum", randomDoubleBetween(-1000, 1000, true));
+ if ((positiveIndices.isEmpty() == false || negativeIndices.isEmpty() == false)) {
+ if (randomBoolean()) {
+ result.put("sum", randomDoubleBetween(-1000, 1000, true));
+ }
+ if (randomBoolean()) {
+ result.put("min", randomDoubleBetween(-1000, 1000, true));
+ }
}
return result;
}
@@ -394,6 +401,11 @@ protected List exampleMalformedValues() {
// Non-Zero sum for empty histogram
exampleMalformedValue(b -> b.startObject().field("scale", 0).field("sum", 42.0).endObject()).errorMatches(
"sum field must be zero if the histogram is empty, but got 42.0"
+ ),
+
+ // Min provided for empty histogram
+ exampleMalformedValue(b -> b.startObject().field("scale", 0).field("min", 42.0).endObject()).errorMatches(
+ "min field must be null if the histogram is empty, but got 42.0"
)
);
}
@@ -442,20 +454,35 @@ private Map convertHistogramToCanonicalForm(Map
List positive = parseBuckets(Types.forciblyCast(histogram.get("positive")));
List negative = parseBuckets(Types.forciblyCast(histogram.get("negative")));
+ Map zeroBucket = convertZeroBucketToCanonicalForm(Types.forciblyCast(histogram.get("zero")));
+
Object sum = histogram.get("sum");
if (sum == null) {
sum = ExponentialHistogramUtils.estimateSum(
- IndexWithCount.asBucketIterator(scale, negative),
- IndexWithCount.asBucketIterator(scale, positive)
+ IndexWithCount.asBuckets(scale, negative).iterator(),
+ IndexWithCount.asBuckets(scale, positive).iterator()
);
}
result.put("sum", sum);
- Map zeroBucket = convertZeroBucketToCanonicalForm(Types.forciblyCast(histogram.get("zero")));
+ Object min = histogram.get("min");
+ if (min == null) {
+ OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+ mapToZeroBucket(zeroBucket),
+ IndexWithCount.asBuckets(scale, negative),
+ IndexWithCount.asBuckets(scale, positive)
+ );
+ if (estimatedMin.isPresent()) {
+ min = estimatedMin.getAsDouble();
+ }
+ }
+ if (min != null) {
+ result.put("min", min);
+ }
+
if (zeroBucket != null) {
result.put("zero", zeroBucket);
}
-
if (positive.isEmpty() == false) {
result.put("positive", writeBucketsInCanonicalForm(positive));
}
@@ -466,6 +493,23 @@ private Map convertHistogramToCanonicalForm(Map
return result;
}
+ private ZeroBucket mapToZeroBucket(Map zeroBucket) {
+ if (zeroBucket == null) {
+ return ZeroBucket.minimalEmpty();
+ }
+ Number threshold = Types.forciblyCast(zeroBucket.get("threshold"));
+ Number count = Types.forciblyCast(zeroBucket.get("count"));
+ if (threshold != null && count != null) {
+ return new ZeroBucket(threshold.doubleValue(), count.longValue());
+ } else if (threshold != null) {
+ return new ZeroBucket(threshold.doubleValue(), 0);
+ } else if (count != null) {
+ return ZeroBucket.minimalWithCount(count.longValue());
+ } else {
+ return ZeroBucket.minimalEmpty();
+ }
+ }
+
private List parseBuckets(Map buckets) {
if (buckets == null) {
return List.of();
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml b/x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml
index c95db7bf8587d..c786fa6c61d1d 100644
--- a/x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml
+++ b/x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml
@@ -29,6 +29,7 @@ setup:
my_histo:
scale: 12
sum: 1234.56
+ min: -345.67
zero:
threshold: 0.123456
count: 42
@@ -47,6 +48,7 @@ setup:
_source.my_histo:
scale: 12
sum: 1234.56
+ min: -345.67
zero:
threshold: 0.123456
count: 42
@@ -68,6 +70,7 @@ setup:
my_histo:
scale: -10
sum: 1234.56
+ min: -345.67
positive:
indices: [1,2,3,4,5]
counts: [6,7,8,9,10]
@@ -83,6 +86,7 @@ setup:
_source.my_histo:
scale: -10
sum: 1234.56
+ min: -345.67
positive:
indices: [1,2,3,4,5]
counts: [6,7,8,9,10]
@@ -101,6 +105,7 @@ setup:
my_histo:
scale: 0
sum: 1234.56
+ min: 345.67
positive:
indices: [-100, 10, 20]
counts: [3, 2, 1]
@@ -113,6 +118,7 @@ setup:
_source.my_histo:
scale: 0
sum: 1234.56
+ min: 345.67
positive:
indices: [-100, 10, 20]
counts: [3, 2, 1]
@@ -128,6 +134,7 @@ setup:
my_histo:
scale: 0
sum: 1234.56
+ min: -345.67
negative:
indices: [-100, 10, 20]
counts: [3, 2, 1]
@@ -140,6 +147,7 @@ setup:
_source.my_histo:
scale: 0
sum: 1234.56
+ min: -345.67
negative:
indices: [-100, 10, 20]
counts: [3, 2, 1]
@@ -206,6 +214,7 @@ setup:
_source.my_histo:
scale: 0
sum: 0.0
+ min: 0.0
zero:
count: 101
@@ -221,6 +230,7 @@ setup:
my_histo:
scale: 38
sum: 1E300
+ min: -1E300
zero:
count: 2305843009213693952 # 2^61 to not cause overflows for the total value count sum
threshold: 1E-300
@@ -238,6 +248,7 @@ setup:
_source.my_histo:
scale: 38
sum: 1E300
+ min: -1E300
zero:
count: 2305843009213693952
threshold: 1E-300
@@ -249,7 +260,7 @@ setup:
counts: [2305843009213693952, 1]
---
-"Sum estimation":
+"Aggregates estimation":
- do:
index:
index: test_exponential_histogram
@@ -273,6 +284,7 @@ setup:
_source.my_histo:
scale: 1
sum: 9.289321881345247
+ min: -2.0
positive:
indices: [8]
counts: [1]
@@ -281,7 +293,7 @@ setup:
counts: [1, 5]
---
-"Positive infinity sum":
+"Positive infinity aggregates":
- do:
index:
index: test_exponential_histogram
@@ -302,12 +314,13 @@ setup:
_source.my_histo:
scale: 0
sum: Infinity
+ min: Infinity
positive:
indices: [2000]
counts: [1]
---
-"negative infinity sum":
+"negative infinity aggregates":
- do:
index:
index: test_exponential_histogram
@@ -328,6 +341,7 @@ setup:
_source.my_histo:
scale: 0
sum: -Infinity
+ min: -Infinity
negative:
indices: [2000]
counts: [1]