Skip to content

SOLR-17806: Migrate DirectUpdateHandler2 metrics to OTEL #3417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jul 29, 2025
Merged

SOLR-17806: Migrate DirectUpdateHandler2 metrics to OTEL #3417

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8ed6c85
Migrate direct update handler
mlbiscoc Jun 30, 2025
58c6eec
Fix up metrics
mlbiscoc Jul 1, 2025
44265eb
UNCOMMIT ME BEFORE PUSHING
mlbiscoc Jul 1, 2025
7752dfd
Revert requestHandlerBase
mlbiscoc Jul 2, 2025
1e43bef
Fix broken tests
mlbiscoc Jul 3, 2025
fc0a690
Cleanup
mlbiscoc Jul 3, 2025
6d2e761
Move commit ops to normal counter
mlbiscoc Jul 9, 2025
eeac8f8
Add new submitted and committed metrics
mlbiscoc Jul 18, 2025
d08dcb3
Move methods around
mlbiscoc Jul 18, 2025
5f344f5
Fix tests for submitted and committed
mlbiscoc Jul 21, 2025
4b21e68
Merge branch 'feature/SOLR-17458' into direct-update-handler
mlbiscoc Jul 24, 2025
15eab76
Use toBuilder() instead of putAll()
mlbiscoc Jul 24, 2025
acf902d
Don't append scope as attribute
mlbiscoc Jul 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
public interface SolrMetricProducer extends AutoCloseable {

public final AttributeKey<String> TYPE_ATTR = AttributeKey.stringKey("type");
public final AttributeKey<String> OPERATION_ATTR = AttributeKey.stringKey("operation");
public final AttributeKey<String> OPERATION_ATTR = AttributeKey.stringKey("ops");

/**
* Unique metric tag identifies components with the same life-cycle, which should be registered /
Expand Down
82 changes: 59 additions & 23 deletions solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ public class DirectUpdateHandler2 extends UpdateHandler
AttributedLongUpDownCounter deleteByIdCommandsCumulative;
AttributedLongUpDownCounter addCommandsCumulative;

AttributedLongCounter submittedAdds;
AttributedLongCounter submittedDeleteById;
AttributedLongCounter submittedDeleteByQuery;

AttributedLongCounter committedAdds;
AttributedLongCounter committedDeleteById;
AttributedLongCounter committedDeleteByQuery;

// Maintenance operations
AttributedLongCounter expungeDeleteCommands;
AttributedLongCounter mergeIndexesCommands;
Expand Down Expand Up @@ -246,8 +254,8 @@ public void initializeMetrics(

var baseCommandsMetric =
solrMetricsContext.longUpDownCounter(
"solr_core_update_operations_cumulative",
"Cumulative number of update commands processed. Metric can go down from rollback command");
"solr_core_update_cumulative_ops",
"Cumulative number of update commands processed. Cumulative can decrease from rollback command");

addCommandsCumulative =
new AttributedLongUpDownCounter(
Expand All @@ -262,29 +270,30 @@ public void initializeMetrics(
baseCommandsMetric,
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_query").build());

var baseCommitMetric =
var baseCommitOpsMetric =
solrMetricsContext.longCounter(
"solr_core_update_commit_operations", "Total number of commit operations");
"solr_core_update_commit_ops", "Total number of commit operations");

commitCommands =
new AttributedLongCounter(
baseCommitMetric, baseAttributes.get().put(OPERATION_ATTR, "commits").build());
baseCommitOpsMetric, baseAttributes.get().put(OPERATION_ATTR, "commits").build());

optimizeCommands =
new AttributedLongCounter(
baseCommitMetric, baseAttributes.get().put(OPERATION_ATTR, "optimize").build());
baseCommitOpsMetric, baseAttributes.get().put(OPERATION_ATTR, "optimize").build());

mergeIndexesCommands =
new AttributedLongCounter(
baseCommitMetric, baseAttributes.get().put(OPERATION_ATTR, "merges").build());
baseCommitOpsMetric, baseAttributes.get().put(OPERATION_ATTR, "merge_indexes").build());

expungeDeleteCommands =
new AttributedLongCounter(
baseCommitMetric, baseAttributes.get().put(OPERATION_ATTR, "expunge_deletes").build());
baseCommitOpsMetric,
baseAttributes.get().put(OPERATION_ATTR, "expunge_deletes").build());

var baseMaintenanceMetric =
solrMetricsContext.longCounter(
"solr_core_update_maintenance_operations", "Total number of maintenance operations");
"solr_core_update_maintenance_ops", "Total number of maintenance operations");

rollbackCommands =
new AttributedLongCounter(
Expand All @@ -302,7 +311,7 @@ public void initializeMetrics(
softAutoCommits =
solrMetricsContext.observableLongCounter(
"solr_core_update_auto_commits",
"Total number of auto commits",
"Current number of auto commits",
(observableLongMeasurement -> {
observableLongMeasurement.record(
commitTracker.getCommitCount(),
Expand All @@ -325,7 +334,6 @@ public void initializeMetrics(
commitTracker.getDocsUpperBound(),
baseAttributes.get().put(TYPE_ATTR, "auto_commit_max_docs").build());
}

if (commitTracker.getTLogFileSizeUpperBound() > 0) {
observableLongMeasurement.record(
commitTracker.getTLogFileSizeUpperBound(),
Expand All @@ -350,22 +358,45 @@ public void initializeMetrics(

updateStats =
solrMetricsContext.observableLongGauge(
"solr_core_update_pending_operations",
"Operations pending commit. Values get reset after commit",
"solr_core_update_docs_pending_commit",
"Current number of documents pending commit. Value is reset to 0 on commit.",
(observableLongMeasurement) -> {
observableLongMeasurement.record(
addCommands.longValue(),
baseAttributes.get().put(OPERATION_ATTR, "adds").build());
observableLongMeasurement.record(
numDocsPending.longValue(),
baseAttributes.get().put(OPERATION_ATTR, "docs_pending").build());
observableLongMeasurement.record(
deleteByIdCommands.longValue(),
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_id").build());
observableLongMeasurement.record(
deleteByQueryCommands.longValue(),
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_query").build());
});

var baseSubmittedOpsMetric =
solrMetricsContext.longCounter(
"solr_core_update_submitted_ops", "Total number of submitted update operations");

var baseCommittedOpsMetric =
solrMetricsContext.longCounter(
"solr_core_update_committed_ops", "Total number of committed update operations");

submittedAdds =
new AttributedLongCounter(
baseSubmittedOpsMetric, baseAttributes.get().put(OPERATION_ATTR, "adds").build());
submittedDeleteById =
new AttributedLongCounter(
baseSubmittedOpsMetric,
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_id").build());
submittedDeleteByQuery =
new AttributedLongCounter(
baseSubmittedOpsMetric,
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_query").build());

committedAdds =
new AttributedLongCounter(
baseCommittedOpsMetric, baseAttributes.get().put(OPERATION_ATTR, "adds").build());
committedDeleteById =
new AttributedLongCounter(
baseCommittedOpsMetric,
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_id").build());
committedDeleteByQuery =
new AttributedLongCounter(
baseCommittedOpsMetric,
baseAttributes.get().put(OPERATION_ATTR, "deletes_by_query").build());
}

private void deleteAll() throws IOException {
Expand Down Expand Up @@ -428,6 +459,7 @@ private int addDoc0(AddUpdateCommand cmd) throws IOException {

addCommands.increment();
addCommandsCumulative.inc();
submittedAdds.inc();

// if there is no ID field, don't overwrite
if (idField == null) {
Expand Down Expand Up @@ -575,6 +607,7 @@ public void delete(DeleteUpdateCommand cmd) throws IOException {
TestInjection.injectDirectUpdateLatch();
deleteByIdCommands.increment();
deleteByIdCommandsCumulative.inc();
submittedDeleteById.inc();

if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0) {
if (ulog != null) ulog.delete(cmd);
Expand Down Expand Up @@ -641,6 +674,7 @@ public void deleteByQuery(DeleteUpdateCommand cmd) throws IOException {
TestInjection.injectDirectUpdateLatch();
deleteByQueryCommands.increment();
deleteByQueryCommandsCumulative.inc();
submittedDeleteByQuery.inc();
boolean madeIt = false;
try {
Query q = getQuery(cmd);
Expand Down Expand Up @@ -896,7 +930,9 @@ public void commit(CommitUpdateCommand cmd) throws IOException {
if (!cmd.softCommit) {
solrCoreState.getCommitLock().unlock();
}

committedAdds.add(addCommands.longValue());
committedDeleteById.add(deleteByIdCommands.longValue());
committedDeleteByQuery.add(deleteByQueryCommands.longValue());
addCommands.reset();
deleteByIdCommands.reset();
deleteByQueryCommands.reset();
Expand Down
46 changes: 17 additions & 29 deletions solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import com.carrotsearch.randomizedtesting.annotations.Repeat;
import com.codahale.metrics.Meter;
import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
import io.prometheus.metrics.model.snapshots.Labels;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
Expand Down Expand Up @@ -576,43 +576,31 @@ public void testOnlyLeaderIndexes() throws Exception {

{
SolrCore core = getSolrCore(true).getFirst();
var reader =
core.getSolrMetricsContext()
.getMetricManager()
.getPrometheusMetricReader(
getSolrCore(true).getFirst().getCoreMetricManager().getRegistryName());
var actual =
(GaugeSnapshot.GaugeDataPointSnapshot)
SolrMetricTestUtils.getDataPointSnapshot(
reader,
"solr_metrics_core_update_pending_operations",
SolrMetricTestUtils.getCloudLabelsBase(core)
.get()
.label("category", "UPDATE")
.label("operation", "docs_pending")
.build());
SolrMetricTestUtils.getGaugeDatapoint(
core,
"solr_core_update_docs_pending_commit",
Labels.builder()
.label("category", "UPDATE")
.label("operation", "docs_pending")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that'd be "ops" now?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was doing a hasSameValues instead of equals which is why it was still passing. Should be fixed now.

.build(),
true);
assertEquals(
"Expected 4 docs are pending in core " + getSolrCore(true).get(0).getCoreDescriptor(),
4,
(long) actual.getValue());
}

for (SolrCore solrCore : getSolrCore(false)) {
var reader =
solrCore
.getSolrMetricsContext()
.getMetricManager()
.getPrometheusMetricReader(solrCore.getCoreMetricManager().getRegistryName());
var actual =
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this statement is lacking some elegance. If this pattern repeats itself; we should seek an improvement

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok so I overhauled the util method a bit for getting metrics in SolrMetricTestUtils. Maybe this is a bit better? If so, I will adopt this paradigm on the other SolrCore PR I did and other PRs going forward.

(GaugeSnapshot.GaugeDataPointSnapshot)
SolrMetricTestUtils.getDataPointSnapshot(
reader,
"solr_metrics_core_update_pending_operations",
SolrMetricTestUtils.getCloudLabelsBase(solrCore)
.get()
.label("category", "UPDATE")
.label("operation", "docs_pending")
.build());
SolrMetricTestUtils.getGaugeDatapoint(
solrCore,
"solr_core_update_docs_pending_commit",
Labels.builder()
.label("category", "UPDATE")
.label("operation", "docs_pending")
.build(),
true);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: booleans like here are inscrutable when reviewing code. Not a big deal though

assertEquals(
"Expected non docs are pending in core " + solrCore.getCoreDescriptor(),
0,
Expand Down
46 changes: 44 additions & 2 deletions solr/core/src/test/org/apache/solr/metrics/SolrMetricTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
import com.codahale.metrics.Counter;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
import io.prometheus.metrics.model.snapshots.CounterSnapshot;
import io.prometheus.metrics.model.snapshots.DataPointSnapshot;
import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
import io.prometheus.metrics.model.snapshots.Labels;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -28,6 +30,7 @@
import java.util.function.Supplier;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;

Expand Down Expand Up @@ -130,7 +133,6 @@ public String toString() {

public static DataPointSnapshot getDataPointSnapshot(
PrometheusMetricReader reader, String metricName, Labels labels) {
var met = reader.collect();
return reader.collect().stream()
.filter(ms -> ms.getMetadata().getPrometheusName().equals(metricName))
.findFirst()
Expand All @@ -156,7 +158,47 @@ public static Supplier<Labels.Builder> getCloudLabelsBase(SolrCore core) {
}

public static Supplier<Labels.Builder> getStandaloneLabelsBase(SolrCore core) {
return getStandaloneLabelsBase(core.getName());
}

public static Supplier<Labels.Builder> getStandaloneLabelsBase(String coreName) {
return () ->
Labels.builder().label("core", core.getName()).label("otel_scope_name", "org.apache.solr");
Labels.builder().label("core", coreName).label("otel_scope_name", "org.apache.solr");
}

public static PrometheusMetricReader getPrometheusMetricReader(SolrCore core) {
return getPrometheusMetricReader(
core.getCoreContainer(), core.getCoreMetricManager().getRegistryName());
}

public static PrometheusMetricReader getPrometheusMetricReader(
CoreContainer container, String registryName) {
return container.getMetricManager().getPrometheusMetricReader(registryName);
}

private static <T> T getDatapoint(
SolrCore core, String metricName, Labels labels, boolean cloudLabels, Class<T> snapshotType) {

var reader = getPrometheusMetricReader(core);

var baseBuilder =
(cloudLabels ? getCloudLabelsBase(core) : getStandaloneLabelsBase(core)).get();

labels.stream().forEach(label -> baseBuilder.label(label.getName(), label.getValue()));

return snapshotType.cast(
SolrMetricTestUtils.getDataPointSnapshot(reader, metricName, baseBuilder.build()));
}

public static GaugeSnapshot.GaugeDataPointSnapshot getGaugeDatapoint(
SolrCore core, String metricName, Labels labels, boolean cloudLabels) {
return getDatapoint(
core, metricName, labels, cloudLabels, GaugeSnapshot.GaugeDataPointSnapshot.class);
}

public static CounterSnapshot.CounterDataPointSnapshot getCounterDatapoint(
SolrCore core, String metricName, Labels labels, boolean cloudLabels) {
return getDatapoint(
core, metricName, labels, cloudLabels, CounterSnapshot.CounterDataPointSnapshot.class);
}
}
Loading