diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py
new file mode 100644
index 000000000..748b70d00
--- /dev/null
+++ b/tests/test_benchmark.py
@@ -0,0 +1,18 @@
+import pytest
+from deepforest import main, get_data
+
+def test_benchmark_release():
+    """
+    Benchmark test to ensure the specific release version of the model
+    produces consistent results.
+    """
+    release_sha = "cc21436bc5d572dde8ff5f93c1e71a32f563cace"
+
+    m = main.deepforest()
+    m.load_model("weecology/deepforest-tree", revision=release_sha)
+
+    csv_file = get_data("OSBS_029.csv")
+    results = m.evaluate(csv_file, iou_threshold=0.4)
+
+    assert results["box_precision"] == pytest.approx(0.8, abs=0.01)
+    assert results["box_recall"] == pytest.approx(0.7213, abs=0.01)
diff --git a/tests/test_main.py b/tests/test_main.py
index a5769ac86..7211091c1 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -559,11 +559,11 @@ def test_evaluate(m):
     df = pd.read_csv(csv_file)
     results = m.evaluate(csv_file)
 
-    # Metrics are sane
-    assert np.round(results["box_precision"], 2) > 0.5
-    assert np.round(results["box_recall"], 2) > 0.5
+    # Check that precision and recall don't regress below reasonable baselines
+    assert results["box_precision"] > 0.7
+    assert results["box_recall"] > 0.5
 
-    # Class names are correct
+    # Structure and Label checks
     assert len(results["results"].predicted_label.dropna().unique()) == 1
     assert results["results"].predicted_label.dropna().unique()[0] == "Tree"
     assert results["predictions"].shape[0] > 0
@@ -578,7 +578,6 @@ def test_evaluate(m):
     # Check we have match results for every ground truth box
     assert results["results"].shape[0] == df.shape[0]
 
-
 def test_train_callbacks(m):
     csv_file = get_data("example.csv")
     root_dir = os.path.dirname(csv_file)