Storing and Plotting Additional QED-C Metrics

.gitignore

@@ -43,3 +43,7 @@ docs/_build/
 
 # mypy
 .mypy_cache/
+
+# QC-App-Oriented-Benchmarks
+__data/
+__images/

metriq_gym/benchmarks/qedc_benchmarks.py

@@ -19,6 +19,7 @@
 from metriq_gym.helpers.task_helpers import flatten_counts
 
 from _common import metrics
+from _common.qiskit import execute as ex
 
 
 QEDC_BENCHMARK_IMPORTS: dict[JobType, str] = {
@@ -41,22 +42,17 @@
 Example for Bernstein-Vazirani:
 {
 '3':    {
-        '1': {'create_time': 0.16371703147888184,
-              'fidelity': 1.0,
+        '1': {'fidelity': 1.0,
               'hf_fidelity': 1.0},
-        '2': {'create_time': 0.0005087852478027344,
-              'fidelity': 1.0,
+        '2': {'fidelity': 1.0,
               'hf_fidelity': 1.0}
         },
 '4':    {
-        '1': {'create_time': 0.0005209445953369141,
-              'fidelity': 1.0,
+        '1': {'fidelity': 1.0,
               'hf_fidelity': 1.0},
-        '3': {'create_time': 0.00047206878662109375,
-              'fidelity': 1.0,
+        '3': {'fidelity': 1.0,
               'hf_fidelity': 1.0},
-        '5': {'create_time': 0.0005078315734863281,
-              'fidelity': 1.0,
+        '5': {'fidelity': 1.0,
               'hf_fidelity': 1.0}
         }
 }
@@ -75,7 +71,7 @@ class QEDCData(BenchmarkData):
                              used to preserve order when polling.
     """
 
-    circuit_metrics: QEDC_Metrics
+    circuit_metrics: QEDC_Metrics | None
     circuit_identifiers: list[tuple[str, str]]
 
 
@@ -141,8 +137,9 @@ def get_counts(self, _):
     benchmark_name = str(params["benchmark_name"])
     benchmark = import_benchmark_module(benchmark_name)
 
-    # Restore circuit metrics dictionary from the dispatch data
-    metrics.circuit_metrics = job_data.circuit_metrics
+    # Restore circuit metrics from the dispatch data if needed
+    if params["extra_metrics"]:
+        metrics.circuit_metrics = job_data.circuit_metrics
 
     # Iterate and get the metrics for each circuit in the list.
     for curr_idx, (num_qubits, circuit_id) in enumerate(job_data.circuit_identifiers):
@@ -171,13 +168,42 @@ def get_counts(self, _):
                 None, result_object, int(num_qubits), int(circuit_id), params["num_shots"]
             )
 
+        # Store the fidelity.
         metrics.store_metric(num_qubits, circuit_id, "fidelity", fidelity)
 
+    # Optionally plotting metrics
+    if params["plot_metrics"]:
+        # Compute statistics for metrics.
+        metrics.aggregate_metrics()
+
+        # Set backend information for plot titles.
+        provider_name = "qBraid"
+        device_name = "<unknown>"
+
+        # Set plot titles.
+        benchmark_title = f"{benchmark_name} ({params.get('method', '1')})"
+        suptitle = f"Benchmark Results - {benchmark_title} - {provider_name}"
+        metrics.circuit_metrics["subtitle"] = f"device = {device_name}"
+
+        # Determine which metrics to plot.
+        if params["extra_metrics"]:
+            filters = ["fidelity", "hf_fidelity", "depth", "2q", "vbplot"]
+        else:
+            metrics.do_volumetric_plots = False
+            filters = ["fidelity", "hf_fidelity"]
+
+        # Plot the metrics.
+        metrics.plot_metrics(suptitle=suptitle, filters=filters)
+
+        # Remove subtilte key.
+        metrics.circuit_metrics.pop("subtitle", None)
+
     return metrics.circuit_metrics
 
 
 def get_circuits_and_metrics(
     benchmark_name: str,
+    extra_metrics: bool,
     params: dict[str, float | str],
 ) -> tuple[list[QuantumCircuit], QEDC_Metrics, list[tuple[str, str]]]:
     """
@@ -204,6 +230,10 @@ def get_circuits_and_metrics(
     # Remove the subtitle key to keep our desired format.
     circuit_metrics.pop("subtitle", None)
 
+    # Optionally copy any initial creation metrics (i.e. create_time).
+    if extra_metrics:
+        metrics.circuit_metrics = circuit_metrics
+
     # Store the circuit identifiers and a flat list of circuits.
     circuit_identifiers = []
     flat_circuits = []
@@ -212,6 +242,23 @@ def get_circuits_and_metrics(
             circuit_identifiers.append((num_qubits, circuit_id))
             flat_circuits.append(circuits[num_qubits][circuit_id])
 
+            # Optionally storing extra metrics.
+            if extra_metrics:
+                # Compute circuit properties (depth, etc) and store to active circuit object
+                ex.compute_and_store_circuit_info(
+                    circuits[num_qubits][circuit_id],
+                    str(num_qubits),
+                    str(circuit_id),
+                    do_transpile_metrics=True,
+                    use_normalized_depth=True,
+                )
+
+    # Optionally returning extra metrics.
+    if extra_metrics:
+        circuit_metrics = metrics.circuit_metrics
+    else:
+        circuit_metrics = None
+
     return flat_circuits, circuit_metrics, circuit_identifiers
 
 
@@ -222,10 +269,14 @@ def dispatch_handler(self, device: QuantumDevice) -> QEDCData:
         # For more information on the parameters, view the schema for this benchmark.
         num_shots = self.params.num_shots
         benchmark_name = self.params.benchmark_name
+        extra_metrics = self.params.extra_metrics
 
         circuits, circuit_metrics, circuit_identifiers = get_circuits_and_metrics(
             benchmark_name=benchmark_name,
-            params=self.params.model_dump(exclude={"benchmark_name"}),
+            extra_metrics=extra_metrics,
+            params=self.params.model_dump(
+                exclude={"benchmark_name", "extra_metrics", "plot_metrics"}
+            ),
         )
 
         return QEDCData.from_quantum_job(

metriq_gym/schemas/bernstein_vazirani.schema.json

@@ -58,6 +58,18 @@
       "type": "integer",
       "description": "Specifies the secret int for the benchmark; note that max_circuits must be 1 if this is used.",
       "examples": [1]
+    },
+    "extra_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to store circuit depth, width, and more metrics.",
+      "default": true,
+      "examples": [false]
+    },
+    "plot_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to plot metrics and store them.",
+      "default": true,
+      "examples": [false]
     }
   },
   "required": ["benchmark_name"],

metriq_gym/schemas/examples/bernstein_vazirani.example.json

@@ -5,5 +5,7 @@
     "max_qubits": 6,
     "skip_qubits": 1,
     "max_circuits": 3,
-    "method": 1
+    "method": 1,
+    "extra_metrics": true,
+    "plot_metrics": true
 }

metriq_gym/schemas/examples/hidden_shift.example.json

@@ -5,5 +5,7 @@
     "max_qubits": 6,
     "skip_qubits": 1,
     "max_circuits": 3,
-    "method": 1
+    "method": 1,
+    "extra_metrics": true,
+    "plot_metrics": true
 }

metriq_gym/schemas/examples/phase_estimation.example.json

@@ -6,5 +6,7 @@
     "skip_qubits": 1,
     "max_circuits": 3,
     "method": 1,
-    "use_midcircuit_measurement": false
+    "use_midcircuit_measurement": false,
+    "extra_metrics": true,
+    "plot_metrics": true
 }

metriq_gym/schemas/examples/quantum_fourier_transform.example.json

@@ -6,5 +6,7 @@
     "skip_qubits": 1,
     "max_circuits": 3,
     "method": 1,
-    "use_midcircuit_measurement": false
+    "use_midcircuit_measurement": false,
+    "extra_metrics": true,
+    "plot_metrics": true
 }

metriq_gym/schemas/hidden_shift.schema.json

@@ -56,6 +56,18 @@
       "type": "integer",
       "description": "Specifies the secret int for the benchmark; note that max_circuits must be 1 if this is used.",
       "examples": [1]
+    },
+    "extra_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to store circuit depth, width, and more metrics.",
+      "default": true,
+      "examples": [false]
+    },
+    "plot_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to plot metrics and store them.",
+      "default": true,
+      "examples": [false]
     }
   },
   "required": ["benchmark_name"],

metriq_gym/schemas/phase_estimation.schema.json

@@ -62,7 +62,18 @@
       "description": "Creates circuits with mid-circuit measurements.",
       "default": false,
       "examples": [true]
-
+    },
+    "extra_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to store circuit depth, width, and more metrics.",
+      "default": true,
+      "examples": [false]
+    },
+    "plot_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to plot metrics and store them.",
+      "default": true,
+      "examples": [false]
     }
   },
   "required": ["benchmark_name"],

metriq_gym/schemas/quantum_fourier_transform.schema.json

@@ -64,7 +64,18 @@
       "description": "Creates circuits with mid-circuit measurements.",
       "default": false,
       "examples": [true]
-
+    },
+    "extra_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to store circuit depth, width, and more metrics.",
+      "default": true,
+      "examples": [false]
+    },
+    "plot_metrics": {
+      "type": "boolean",
+      "description": "Determines whether to plot metrics and store them.",
+      "default": true,
+      "examples": [false]
     }
   },
   "required": ["benchmark_name"],