From fed9fdf1065cc1ce263dbaf9d9870f325c50aba3 Mon Sep 17 00:00:00 2001
From: "google-labs-jules[bot]"
 <161369871+google-labs-jules[bot]@users.noreply.github.com>
Date: Thu, 25 Jun 2026 09:34:54 +0000
Subject: [PATCH] perf: optimize DataFrame iteration in
 verify_processed_omol25.py

Replace `df.iterrows()` with `df.to_dict('records')` to significantly speed up DataFrame iteration, eliminating the overhead of Pandas creating a Series object for each row. Update downstream variables to reflect that row items are now native Python dicts rather than Pandas Series objects. Added inline comments to explain the rationale.

Co-authored-by: alinelena <3306823+alinelena@users.noreply.github.com>
---
 .jules/bolt.md                               |  3 +++
 src/lavello_mlips/verify_processed_omol25.py | 12 +++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/.jules/bolt.md b/.jules/bolt.md
index d87bd6d..6252ff3 100644
--- a/.jules/bolt.md
+++ b/.jules/bolt.md
@@ -8,3 +8,6 @@
 ## 2024-03-29 - ASE Custom JSON encoding vs standard JSON
 **Learning:** ASE's custom JSON encoder (`ase.io.jsonio.encode`) will generate dicts with special keys like `__ndarray__` or `__complex__` (e.g. `{"__ndarray__": [[5], "int64", ...]}`). When optimizing JSON deserialization using faster alternatives like `orjson`, it's critical to realize that a normal `json.loads` or `orjson.loads` will deserialize this into a Python dictionary, while ASE's custom `decode` will properly reconstruct the underlying numpy array. Bypassing ASE's decoder without checking for these keys leads to downstream type errors (e.g. `KeyError: '__ndarray__'`).
 **Action:** When replacing or wrapping ASE's jsonio with `orjson`, always fall back to ASE's `decode` if the payload string contains `__ndarray__` or `__complex__` markers, to ensure custom objects are correctly reconstructed.
+## 2026-06-25 - Optimizing Pandas DataFrame iteration
+**Learning:** In Pandas, iterating over a DataFrame using `df.iterrows()` is notoriously slow due to the overhead of creating a new Series object for each row. A highly effective, often 10x-100x faster alternative for large datasets is to convert the DataFrame to a list of native Python dictionaries first using `df.to_dict('records')`, and then iterate over that list.
+**Action:** When you need to iterate over rows in a Pandas DataFrame, avoid `iterrows()`. Instead, use `df.to_dict('records')` to convert the data into a list of dictionaries before looping. Always update downstream references, as the rows will now be native dictionaries, not Series objects.
diff --git a/src/lavello_mlips/verify_processed_omol25.py b/src/lavello_mlips/verify_processed_omol25.py
index 9d1c47c..bf755e2 100644
--- a/src/lavello_mlips/verify_processed_omol25.py
+++ b/src/lavello_mlips/verify_processed_omol25.py
@@ -77,8 +77,14 @@ def main() -> None:
         )
 
     logger.info(f"Loaded {len(df)} records from Parquet.")
-    parquet_by_sha = {row["geom_sha1"]: row for _, row in df.iterrows()}
-    parquet_by_argone_rel = {row["argonne_rel"]: row for _, row in df.iterrows()}
+
+    # Performance Optimization: Convert DataFrame to a list of dicts first.
+    # Iterating over df.to_dict("records") is significantly faster than df.iterrows(),
+    # which creates a new Pandas Series for every single row.
+    records = df.to_dict("records")
+    parquet_by_sha = {row["geom_sha1"]: row for row in records}
+    parquet_by_argone_rel = {row["argonne_rel"]: row for row in records}
+
     logger.info(f"Loading ExtXYZ file from {args.extxyz} (this may take a moment)...")
     all_atoms = read(str(args.extxyz), index=":")
     if not isinstance(all_atoms, list):
@@ -108,7 +114,7 @@ def get_dump_entry(at):
             info = dict(at.info)
             rel = info.get("argonne_rel")
             pq_row = parquet_by_argone_rel.get(rel)
-            pq_data = pq_row.to_dict() if pq_row is not None else None
+            pq_data = pq_row if pq_row is not None else None
             return {"xyz": info, "parquet": pq_data}
 
         duplicates = {