OpenNMS · cnewkirk · Mar 29, 2026
diff --git a/smoke-test/README.md b/smoke-test/README.md
@@ -73,6 +73,97 @@ Smoke tests will run against this image in your local Docker image repo.
 When you want to use the latest **published** Horizon docker image, make sure to remove the local image with `docker image rm opennms/horizon:latest`.
 You can do the same for the Minion and Sentinel.
 
+## Time Series Strategy: INTEGRATION
+
+The smoke test infrastructure supports pluggable time series storage (TSS) backends via `TimeSeriesStrategy.INTEGRATION`. This enables end-to-end testing of TSS plugins (e.g., the Cortex TSS plugin) with real Prometheus-compatible backends.
+
+### Architecture
+
+```mermaid
+graph LR
+    subgraph TestContainers Stack
+        PG[PostgreSQL]
+        TR[Thanos Receive<br/>:19291 write<br/>:10901 gRPC<br/>:10902 HTTP]
+        TQ[Thanos Query<br/>:9090 HTTP<br/>:10903 gRPC]
+        ONMS[OpenNMS Horizon<br/>:8980 Web/REST<br/>:8101 Karaf SSH]
+    end
+
+    subgraph OpenNMS Internals
+        COL[Collectd] -->|samples| TSS[TimeSeries<br/>Storage Manager]
+        TSS -->|Prom remote write| CORTEX[Cortex TSS Plugin]
+        MEAS[Measurements API] -->|Prom query API| CORTEX
+    end
+
+    CORTEX -->|write| TR
+    CORTEX -->|read| TQ
+    TQ -->|gRPC store| TR
+    ONMS --- PG
+
+    TEST[CortexTssPluginIT] -->|REST API| ONMS
+    TEST -->|Prom query API| TQ
+```
+
+### Data Flow
+
+1. **Write path**: OpenNMS Collectd collects JVM/DB/SNMP metrics every 30s (self-monitor node) -> `TimeseriesStorageManager` -> Cortex TSS plugin -> Prometheus remote write to Thanos Receive
+2. **Read path**: Test queries OpenNMS Measurements API -> Cortex TSS plugin -> Prometheus query API on Thanos Query -> data returned through OpenNMS REST response
+
+### Prerequisites
+
+The Cortex TSS plugin is built separately from OpenNMS. You need the plugin KAR file before running these tests.
+
+```bash
+# 1. Clone and build the plugin
+git clone https://github.com/OpenNMS/opennms-cortex-tss-plugin.git
+cd opennms-cortex-tss-plugin
+mvn clean install -DskipTests
+
+# 2. Run the smoke tests with the KAR path
+cd /path/to/opennms
+./compile.pl -t -Dsmoke -Dsmoke.flaky \
+    -Dcortex.kar=/path/to/opennms-cortex-tss-plugin/assembly/kar/target/opennms-cortex-tss-plugin.kar \
+    --projects :smoke-test install
+
+# Alternative: bind-mount your entire Maven repo into the container
+./compile.pl -t -Dsmoke -Dsmoke.flaky \
+    -Dorg.opennms.dev.m2=$HOME/.m2/repository \
+    --projects :smoke-test install
+```
+
+### Container Setup
+
+```java
+@ClassRule
+public static OpenNMSStack stack = OpenNMSStack.withModel(StackModel.newBuilder()
+        .withTimeSeriesStrategy(TimeSeriesStrategy.INTEGRATION)
+        .withOpenNMS(OpenNMSProfile.newBuilder()
+                .withInstallFeature("opennms-plugins-cortex-tss", "opennms-cortex-tss-plugin")
+                .build())
+        .build());
+```
+
+This wires up:
+- **ThanosReceiveContainer** (write endpoint) and **ThanosQueryContainer** (read endpoint) in the RuleChain
+- Cortex plugin config (`org.opennms.plugins.tss.cortex.cfg`) pointing at the Thanos containers
+- `org.opennms.timeseries.strategy=integration` in system properties
+- `opennms-timeseries-api` feature in Karaf boot
+
+### Validation Utilities
+
+`TimeSeriesValidationUtils` provides strategy-agnostic validation methods reusable with any TSS backend:
+
+- `validateResourceTree(restClient, nodeCriteria)` — verifies the resource tree is populated with child resources
+- `validateMeasurements(restClient, resourceId, attribute, aggregation)` — verifies data round-trips through the measurements API
+- `validateMeasurementsMetadata(restClient, resourceId, attribute)` — verifies measurement metadata (node labels, resource info)
+
+### Available Containers
+
+| Container | Class | Default Ports | Purpose |
+|-----------|-------|---------------|---------|
+| Thanos Receive | `ThanosReceiveContainer` | 19291 (write), 10901 (gRPC), 10902 (HTTP) | Accepts Prometheus remote write |
+| Thanos Query | `ThanosQueryContainer` | 9090 (HTTP), 10903 (gRPC) | Prometheus-compatible query API |
+| Prometheus | `PrometheusContainer` | 9090 (HTTP) | Standalone building block (not used by default INTEGRATION stack) |
+
 ## Writing system tests
 
 When writing a new test, use the stack rule to setup the environment:

diff --git a/smoke-test/src/main/java/org/opennms/smoketest/containers/OpenNMSContainer.java b/smoke-test/src/main/java/org/opennms/smoketest/containers/OpenNMSContainer.java
@@ -92,6 +92,8 @@ public class OpenNMSContainer extends GenericContainer<OpenNMSContainer> impleme
     public static final String KAFKA_ALIAS = "kafka";
     public static final String ELASTIC_ALIAS = "elastic";
     public static final String CASSANDRA_ALIAS = "cassandra";
+    public static final String THANOS_RECEIVE_ALIAS = ThanosReceiveContainer.ALIAS;
+    public static final String THANOS_QUERY_ALIAS = ThanosQueryContainer.ALIAS;
 
     public static final String ADMIN_USER = "admin";
     public static final String ADMIN_PASSWORD = "admin";
@@ -152,7 +154,8 @@ public OpenNMSContainer(StackModel model, OpenNMSProfile profile) {
         this.overlay = writeOverlay();
 
         String containerCommand = "-s";
-        if (TimeSeriesStrategy.NEWTS.equals(model.getTimeSeriesStrategy())) {
+        if (TimeSeriesStrategy.NEWTS.equals(model.getTimeSeriesStrategy())
+                || TimeSeriesStrategy.INTEGRATION.equals(model.getTimeSeriesStrategy())) {
             this.withEnv("OPENNMS_TIMESERIES_STRATEGY", model.getTimeSeriesStrategy().name().toLowerCase());
         }
 
@@ -313,6 +316,24 @@ private void writeOverlay(Path home) throws IOException {
                             .put("compression.type", model.getKafkaCompressionStrategy().getCodec())
                             .build());
         }
+
+        // Currently assumes Cortex TSS plugin; generalize when additional TSS plugins need smoke testing
+        if (TimeSeriesStrategy.INTEGRATION.equals(model.getTimeSeriesStrategy())) {
+            writeProps(etc.resolve("org.opennms.plugins.tss.cortex.cfg"),
+                    ImmutableMap.<String,String>builder()
+                            .put("writeUrl", "http://" + THANOS_RECEIVE_ALIAS + ":" + ThanosReceiveContainer.REMOTE_WRITE_PORT + "/api/v1/receive")
+                            .put("readUrl", "http://" + THANOS_QUERY_ALIAS + ":" + ThanosQueryContainer.HTTP_PORT + "/api/v1")
+                            .put("maxConcurrentHttpConnections", "100") // generous for smoke test parallelism
+                            .put("writeTimeoutInMs", "5000")
+                            .put("readTimeoutInMs", "30000") // Thanos queries can be slow on first compaction
+                            .put("metricCacheSize", "1000")
+                            .put("externalTagsCacheSize", "1000")
+                            .put("bulkheadMaxWaitDuration", String.valueOf(Long.MAX_VALUE)) // disable timeout — smoke tests should not shed load
+                            .put("maxSeriesLookback", "31536000") // 365 days in seconds — look back far enough to find all test data
+                            .put("useLabelValuesForDiscovery", "true")
+                            .put("discoveryBatchSize", "50") // batch size for label-values two-phase discovery
+                            .build());
+        }
     }
 
     /**
@@ -400,6 +421,11 @@ public Properties getSystemProperties() {
             props.put("org.opennms.newts.config.hostname", CASSANDRA_ALIAS);
             props.put("org.opennms.newts.config.port", Integer.toString(CassandraContainer.CQL_PORT));
             props.put("org.opennms.rrd.storeByForeignSource", Boolean.TRUE.toString());
+        } else if (TimeSeriesStrategy.INTEGRATION.equals(model.getTimeSeriesStrategy())) {
+            // Use the Integration API with a TSS plugin (e.g. Cortex/Thanos)
+            props.put("org.opennms.timeseries.strategy", "integration");
+            props.put("org.opennms.timeseries.tin.metatags.tag.node", "${node:label}");
+            props.put("org.opennms.timeseries.tin.metatags.tag.location", "${node:location}");
         }
 
         if (model.isJaegerEnabled()) {
@@ -436,6 +462,9 @@ public List<String> getFeaturesOnBoot() {
         if (model.isJaegerEnabled()) {
             featuresOnBoot.add("opennms-core-tracing-jaeger");
         }
+        if (TimeSeriesStrategy.INTEGRATION.equals(model.getTimeSeriesStrategy())) {
+            featuresOnBoot.add("opennms-timeseries-api");
+        }
         return featuresOnBoot;
     }
 

diff --git a/smoke-test/src/main/java/org/opennms/smoketest/containers/PrometheusContainer.java b/smoke-test/src/main/java/org/opennms/smoketest/containers/PrometheusContainer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.smoketest.containers;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.opennms.smoketest.utils.TestContainerUtils;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Vanilla Prometheus container with remote write receiver enabled.
+ * Provides the gold-standard Prometheus API for integration tests.
+ *
+ * <p>This container is not started by the default INTEGRATION stack (which uses
+ * Thanos Receive + Query instead). It is available as a standalone building block
+ * for custom test stacks that want a simpler single-node Prometheus backend.</p>
+ */
+public class PrometheusContainer extends GenericContainer<PrometheusContainer> {
+
+    public static final String ALIAS = "prometheus";
+    public static final int HTTP_PORT = 9090;
+
+    public PrometheusContainer() {
+        super(DockerImageName.parse("docker.io/prom/prometheus:v2.53.4"));
+        withCommand(
+                "--config.file=/etc/prometheus/prometheus.yml",
+                "--storage.tsdb.path=/prometheus",
+                "--storage.tsdb.retention.time=30d",
+                "--web.enable-remote-write-receiver",
+                "--web.listen-address=0.0.0.0:" + HTTP_PORT
+        );
+        withClasspathResourceMapping("prometheus.yml", "/etc/prometheus/prometheus.yml",
+                BindMode.READ_ONLY);
+        withExposedPorts(HTTP_PORT);
+        withNetwork(Network.SHARED);
+        withNetworkAliases(ALIAS);
+        withCreateContainerCmdModifier(TestContainerUtils::setGlobalMemAndCpuLimits);
+    }
+
+    /**
+     * @return the internal write URL for Prometheus remote write (for use by containers on the shared network)
+     */
+    public String getInternalWriteUrl() {
+        return String.format("http://%s:%d/api/v1/write", ALIAS, HTTP_PORT);
+    }
+
+    /**
+     * @return the internal read URL for the Prometheus query API (for use by containers on the shared network)
+     */
+    public String getInternalReadUrl() {
+        return String.format("http://%s:%d/api/v1", ALIAS, HTTP_PORT);
+    }
+
+    /**
+     * @return the external URL for the Prometheus query API (for use by the test host)
+     */
+    public URL getExternalQueryUrl() {
+        try {
+            return new URL(String.format("http://%s:%d", getContainerIpAddress(), getMappedPort(HTTP_PORT)));
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/smoke-test/src/main/java/org/opennms/smoketest/containers/ThanosQueryContainer.java b/smoke-test/src/main/java/org/opennms/smoketest/containers/ThanosQueryContainer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.smoketest.containers;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.opennms.smoketest.utils.TestContainerUtils;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Thanos Query container that provides a Prometheus-compatible query API.
+ * Used as the read endpoint for the Cortex TSS plugin in integration tests.
+ */
+public class ThanosQueryContainer extends GenericContainer<ThanosQueryContainer> {
+
+    public static final String ALIAS = "thanos-query";
+    public static final int HTTP_PORT = 9090;
+    public static final int GRPC_PORT = 10903;
+
+    public ThanosQueryContainer() {
+        super(DockerImageName.parse("docker.io/thanosio/thanos:v0.35.1"));
+        withCommand(
+                "query",
+                "--http-address=0.0.0.0:" + HTTP_PORT,
+                "--grpc-address=0.0.0.0:" + GRPC_PORT,
+                "--store=" + ThanosReceiveContainer.ALIAS + ":" + ThanosReceiveContainer.GRPC_PORT
+        );
+        withExposedPorts(HTTP_PORT, GRPC_PORT);
+        withNetwork(Network.SHARED);
+        withNetworkAliases(ALIAS);
+        withCreateContainerCmdModifier(TestContainerUtils::setGlobalMemAndCpuLimits);
+    }
+
+    /**
+     * @return the internal URL for the Prometheus query API (for use by containers on the shared network)
+     */
+    public String getInternalReadUrl() {
+        return String.format("http://%s:%d/api/v1", ALIAS, HTTP_PORT);
+    }
+
+    /**
+     * @return the external URL for the Prometheus query API (for use by the test host)
+     */
+    public URL getExternalQueryUrl() {
+        try {
+            return new URL(String.format("http://%s:%d", getContainerIpAddress(), getMappedPort(HTTP_PORT)));
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/smoke-test/src/main/java/org/opennms/smoketest/containers/ThanosReceiveContainer.java b/smoke-test/src/main/java/org/opennms/smoketest/containers/ThanosReceiveContainer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.smoketest.containers;
+
+import org.opennms.smoketest.utils.TestContainerUtils;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Thanos Receive container that accepts Prometheus remote write requests.
+ * Used as the write endpoint for the Cortex TSS plugin in integration tests.
+ */
+public class ThanosReceiveContainer extends GenericContainer<ThanosReceiveContainer> {
+
+    public static final String ALIAS = "thanos-receive";
+    public static final int REMOTE_WRITE_PORT = 19291;
+    public static final int GRPC_PORT = 10901;
+    public static final int HTTP_PORT = 10902;
+
+    public ThanosReceiveContainer() {
+        super(DockerImageName.parse("docker.io/thanosio/thanos:v0.35.1"));
+        withCommand(
+                "receive",
+                "--tsdb.path=/data",
+                "--grpc-address=0.0.0.0:" + GRPC_PORT,
+                "--http-address=0.0.0.0:" + HTTP_PORT,
+                "--remote-write.address=0.0.0.0:" + REMOTE_WRITE_PORT,
+                "--tsdb.retention=30d",
+                "--label=receive_replica=\"0\""
+        );
+        withExposedPorts(REMOTE_WRITE_PORT, GRPC_PORT, HTTP_PORT);
+        withNetwork(Network.SHARED);
+        withNetworkAliases(ALIAS);
+        withCreateContainerCmdModifier(TestContainerUtils::setGlobalMemAndCpuLimits);
+    }
+
+    /**
+     * @return the internal URL for Prometheus remote write (for use by containers on the shared network)
+     */
+    public String getInternalRemoteWriteUrl() {
+        return String.format("http://%s:%d/api/v1/receive", ALIAS, REMOTE_WRITE_PORT);
+    }
+}