Workflow API Improvements

src/ptychodus/api/object.py

@@ -203,6 +203,9 @@ def layer_spacing_m(self) -> Sequence[float]:
     def get_total_thickness_m(self) -> float:
         return sum(self._layer_spacing_m)
 
+    def __repr__(self) -> str:
+        return f'{self._array.dtype}{self._array.shape}'
+
 
 class ObjectFileReader(ABC):
     @abstractmethod

src/ptychodus/api/parametric.py

@@ -211,6 +211,9 @@ def get_value(self) -> float:
 
         return value
 
+    def set_value(self, value: float, *, notify: bool = True) -> None:
+        super().set_value(float(value), notify=notify)
+
     def set_value_from_string(self, value: str) -> None:
         self.set_value(float(value))
 

src/ptychodus/api/probe.py

@@ -169,10 +169,19 @@ def __init__(
             self._opr_weights = None
         elif numpy.issubdtype(opr_weights.dtype, numpy.floating):
             if opr_weights.ndim == 2:
-                if opr_weights.shape[1] == self._array.shape[0]:
+                num_weights_actual = opr_weights.shape[1]
+                num_weights_expected = self._array.shape[0]
+
+                if num_weights_actual == num_weights_expected:
                     self._opr_weights = opr_weights
                 else:
-                    raise ValueError('opr_weights do not match the number of coherent probe modes')
+                    raise ValueError(
+                        (
+                            'inconsistent number of opr weights!'
+                            f' actual={num_weights_actual}'
+                            f' expected={num_weights_expected}'
+                        )
+                    )
             else:
                 raise ValueError('opr_weights must be 2-dimensional ndarray')
         else:
@@ -267,6 +276,9 @@ def get_geometry(self) -> ProbeGeometry:
     def __len__(self) -> int:
         return 1 if self._opr_weights is None else self._opr_weights.shape[0]
 
+    def __repr__(self) -> str:
+        return f'{self._array.dtype}{self._array.shape}'
+
 
 class ProbeFileReader(ABC):
     @abstractmethod

src/ptychodus/api/scan.py

@@ -87,6 +87,9 @@ def __len__(self) -> int:
     def nbytes(self) -> int:
         return self._coordinates_m.nbytes
 
+    def __repr__(self) -> str:
+        return f'{self._coordinates_m.dtype}{self._coordinates_m.shape}'
+
 
 class ScanPointParseError(Exception):
     """raised when the scan file cannot be parsed"""

src/ptychodus/api/workflow.py

@@ -10,6 +10,10 @@
 
 
 class WorkflowProductAPI(ABC):
+    @abstractmethod
+    def get_product_index(self) -> int:
+        pass
+
     @abstractmethod
     def open_scan(self, file_path: Path, *, file_type: str | None = None) -> None:
         pass
@@ -41,7 +45,7 @@ def build_object(
         pass
 
     @abstractmethod
-    def reconstruct_local(self) -> WorkflowProductAPI:
+    def reconstruct_local(self, block: bool = False) -> WorkflowProductAPI:
         pass
 
     @abstractmethod
@@ -52,6 +56,10 @@ def reconstruct_remote(self) -> None:
     def save_product(self, file_path: Path, *, file_type: str | None = None) -> None:
         pass
 
+    @abstractmethod
+    def export_training_data(self, file_path: Path) -> None:
+        pass
+
 
 class WorkflowAPI(ABC):
     @abstractmethod
@@ -76,6 +84,11 @@ def export_assembled_patterns(self, file_path: Path) -> None:
         """export assembled patterns"""
         pass
 
+    @abstractmethod
+    def get_product(self, product_index: int) -> WorkflowProductAPI:
+        """returns a product by index"""
+        pass
+
     @abstractmethod
     def open_product(self, file_path: Path, *, file_type: str | None = None) -> WorkflowProductAPI:
         """opens product from file"""
@@ -103,6 +116,10 @@ def save_settings(
     ) -> None:
         pass
 
+    @abstractmethod
+    def set_reconstructor(self, reconstructor_name: str) -> None:
+        pass
+
 
 class FileBasedWorkflow(ABC):
     @property

src/ptychodus/model/core.py

@@ -231,7 +231,6 @@ def batch_mode_execute(
         input_path: Path,
         output_path: Path,
         *,
-        product_file_type: str = 'NPZ',
         fluorescence_input_file_path: Path | None = None,
         fluorescence_output_file_path: Path | None = None,
     ) -> int:
@@ -241,39 +240,20 @@ def batch_mode_execute(
             self.reconstructor.reconstructor_api.save_model(output_path)
             return output.result
 
-        input_product_index = self.product.product_api.open_product(
-            input_path, file_type=product_file_type
-        )
-
-        if input_product_index < 0:
-            logger.error(f'Failed to open product "{input_path}"!')
-            return -1
-
         if action.lower() == 'reconstruct':
-            logger.info('Reconstructing...')
-            output_product_index = self.reconstructor.reconstructor_api.reconstruct(
-                input_product_index
-            )
-            self.reconstructor.reconstructor_api.process_results(block=True)
-            logger.info('Reconstruction complete.')
-
-            self.product.product_api.save_product(
-                output_product_index, output_path, file_type=product_file_type
-            )
+            input_product_api = self.workflow.workflow_api.open_product(input_path)
+            output_product_api = input_product_api.reconstruct_local(block=True)
+            output_product_api.save_product(output_path)
 
             if (
                 fluorescence_input_file_path is not None
                 and fluorescence_output_file_path is not None
             ):
                 self.fluorescence_core.enhance_fluorescence(
-                    output_product_index,
+                    output_product_api.get_product_index(),
                     fluorescence_input_file_path,
                     fluorescence_output_file_path,
                 )
-        elif action.lower() == 'prepare_training_data':
-            self.reconstructor.reconstructor_api.export_training_data(
-                output_path, input_product_index
-            )
         else:
             logger.error(f'Unknown batch mode action "{action}"!')
             return -1

src/ptychodus/model/patterns/core.py

@@ -47,12 +47,13 @@ def start(self) -> None:
         pass
 
     def stop(self) -> None:
-        self.dataset.finish_loading(block=False)
+        self.patterns_api.finish_assembling_diffraction_patterns(block=False)
 
     def _update(self, observable: Observable) -> None:
         if observable is self._reinit_observable:
             self.patterns_api.open_patterns(
                 file_path=self.pattern_settings.file_path.get_value(),
                 file_type=self.pattern_settings.file_type.get_value(),
             )
-            self.dataset.start_loading()
+            self.patterns_api.start_assembling_diffraction_patterns()
+            self.patterns_api.finish_assembling_diffraction_patterns(block=True)

src/ptychodus/model/product/item_factory.py

@@ -136,7 +136,7 @@ def create_from_settings(self) -> ProductRepositoryItem:
         probe_item = self._probe_item_factory.create_from_settings(geometry)
         object_item = self._object_item_factory.create_from_settings(geometry)
 
-        return ProductRepositoryItem(
+        item = ProductRepositoryItem(
             parent=self._repository,
             metadata_item=metadata_item,
             scan_item=scan_item,
@@ -146,3 +146,5 @@ def create_from_settings(self) -> ProductRepositoryItem:
             validator=ProductValidator(self._dataset, scan_item, geometry, probe_item, object_item),
             costs=list(),
         )
+        logger.debug(f'Created product from settings: {item.get_name()}')
+        return item

src/ptychodus/model/ptychi/helper.py

@@ -697,16 +697,14 @@ def create_product(
 
         corrected_scan_points: list[ScanPoint] = list()
         object_geometry = object_in.get_geometry()
-        rx_px = object_geometry.width_px / 2
-        ry_px = object_geometry.height_px / 2
 
         for uncorrected_point, pos_x_px, pos_y_px in zip(
             product.positions, position_x_px, position_y_px
         ):
             object_point = ObjectPoint(
                 index=uncorrected_point.index,
-                position_x_px=float(pos_x_px + rx_px),
-                position_y_px=float(pos_y_px + ry_px),
+                position_x_px=float(pos_x_px),
+                position_y_px=float(pos_y_px),
             )
             scan_point = object_geometry.map_object_point_to_scan_point(object_point)
             corrected_scan_points.append(scan_point)

src/ptychodus/model/reconstructor/api.py

@@ -155,3 +155,7 @@ def train(self, data_path: Path) -> TrainOutput:
             logger.warning('Reconstructor is not trainable!')
 
         return result
+
+    def set_reconstructor(self, name: str) -> str:
+        self._reconstructor_chooser.set_current_plugin(name)
+        return self._reconstructor_chooser.get_current_plugin().simple_name

src/ptychodus/model/reconstructor/matcher.py

@@ -53,7 +53,7 @@ def match_diffraction_patterns_with_positions(
     ) -> ReconstructInput:
         input_product_item = self._product_api.get_item(input_product_index)
         input_product = input_product_item.get_product()
-        data_indexes = self._dataset.get_assembled_indexes()
+        data_indexes = [int(index) for index in self._dataset.get_assembled_indexes()]
         scan_indexes = [
             point.index for point in input_product.positions if index_filter(point.index)
         ]

src/ptychodus/model/reconstructor/queue.py

@@ -83,11 +83,15 @@ def _reconstruct(self) -> None:
                 input_task = self._input_queue.get(block=True, timeout=1)
 
                 try:
+                    logger.debug('Reconstructing...')
                     output_task = input_task.execute()
+                    logger.debug('Reconstruction finished.')
                 except Exception:
                     logger.exception('Reconstructor error!')
                 else:
+                    logger.debug('Adding reconstruction result to output queue...')
                     self._output_queue.put(output_task)
+                    logger.debug('Reconstruction result added to output queue.')
                 finally:
                     self._input_queue.task_done()
             except queue.Empty:
@@ -102,17 +106,27 @@ def put(
         task = ExecuteReconstructorTask(
             self._data_matcher, reconstructor, product_index, index_filter
         )
+        logger.debug('Adding reconstruction task to queue...')
         self._input_queue.put(task)
+        logger.debug('Reconstruction task added to queue.')
 
     def process_results(self, *, block: bool) -> None:
         while True:
             try:
+                logger.debug('Waiting for reconstruction result...')
                 task = self._output_queue.get(block=block)
+                logger.debug('Reconstruction result received.')
 
                 try:
+                    logger.debug('Processing reconstruction result...')
                     task.execute()
+                    logger.debug('Reconstruction result processed.')
                 finally:
                     self._output_queue.task_done()
+
+                    if block and self._output_queue.empty():
+                        logger.debug('No more results to process.')
+                        break
             except queue.Empty:
                 break
 

src/ptychodus/model/workflow/api.py

@@ -36,6 +36,9 @@ def __init__(
         self._executor = executor
         self._product_index = product_index
 
+    def get_product_index(self) -> int:
+        return self._product_index
+
     def open_scan(self, file_path: Path, *, file_type: str | None = None) -> None:
         self._scan_api.open_scan(self._product_index, file_path, file_type=file_type)
 
@@ -69,9 +72,11 @@ def build_object(
         else:
             self._object_api.build_object(self._product_index, builder_name, builder_parameters)
 
-    def reconstruct_local(self) -> WorkflowProductAPI:
-        logger.debug(f'Reconstruct: index={self._product_index}')
+    def reconstruct_local(self, block: bool = False) -> WorkflowProductAPI:
+        logger.info('Reconstructing...')
         output_product_index = self._reconstructor_api.reconstruct(self._product_index)
+        self._reconstructor_api.process_results(block=block)
+        logger.info('Reconstruction complete.')
 
         return ConcreteWorkflowProductAPI(
             self._product_api,
@@ -90,6 +95,9 @@ def reconstruct_remote(self) -> None:
     def save_product(self, file_path: Path, *, file_type: str | None = None) -> None:
         self._product_api.save_product(self._product_index, file_path, file_type=file_type)
 
+    def export_training_data(self, file_path: Path) -> None:
+        self._reconstructor_api.export_training_data(file_path, self._product_index)
+
 
 class ConcreteWorkflowAPI(WorkflowAPI):
     def __init__(
@@ -112,20 +120,6 @@ def __init__(
         self._reconstructor_api = reconstructor_api
         self._executor = executor
 
-    def _create_product_api(self, product_index: int) -> WorkflowProductAPI:
-        if product_index < 0:
-            raise ValueError(f'Bad product index ({product_index=})!')
-
-        return ConcreteWorkflowProductAPI(
-            self._product_api,
-            self._scan_api,
-            self._probe_api,
-            self._object_api,
-            self._reconstructor_api,
-            self._executor,
-            product_index,
-        )
-
     def open_patterns(
         self,
         file_path: Path,
@@ -144,9 +138,27 @@ def import_assembled_patterns(self, file_path: Path) -> None:
     def export_assembled_patterns(self, file_path: Path) -> None:
         self._patterns_api.export_assembled_patterns(file_path)
 
+    def get_product(self, product_index: int) -> WorkflowProductAPI:
+        if product_index < 0:
+            raise ValueError(f'Bad product index ({product_index=})!')
+
+        return ConcreteWorkflowProductAPI(
+            self._product_api,
+            self._scan_api,
+            self._probe_api,
+            self._object_api,
+            self._reconstructor_api,
+            self._executor,
+            product_index,
+        )
+
     def open_product(self, file_path: Path, *, file_type: str | None = None) -> WorkflowProductAPI:
         product_index = self._product_api.open_product(file_path, file_type=file_type)
-        return self._create_product_api(product_index)
+
+        if product_index < 0:
+            raise RuntimeError(f'Failed to open product "{file_path}"!')
+
+        return self.get_product(product_index)
 
     def create_product(
         self,
@@ -170,9 +182,13 @@ def create_product(
             mass_attenuation_m2_kg=mass_attenuation_m2_kg,
             tomography_angle_deg=tomography_angle_deg,
         )
-        return self._create_product_api(product_index)
+        return self.get_product(product_index)
 
     def save_settings(
         self, file_path: Path, change_path_prefix: PathPrefixChange | None = None
     ) -> None:
         self._settings_registry.save_settings(file_path, change_path_prefix)
+
+    def set_reconstructor(self, reconstructor_name: str) -> None:
+        reconstructor = self._reconstructor_api.set_reconstructor(reconstructor_name)
+        logger.debug(f'{reconstructor=}')

src/ptychodus/plugins/npz_diffraction_file.py

@@ -79,7 +79,7 @@ def write(self, file_path: Path, dataset: DiffractionDataset) -> None:
             self.INDEXES: numpy.concatenate([array.get_indexes() for array in dataset]),
             self.PATTERNS: numpy.concatenate([array.get_data() for array in dataset]),
         }
-        numpy.savez(file_path, **contents)
+        numpy.savez_compressed(file_path, **contents)
 
 
 def register_plugins(registry: PluginRegistry) -> None:

src/ptychodus/plugins/xrf_maps_file.py

@@ -85,7 +85,7 @@ def write(self, file_path: Path, dataset: FluorescenceDataset) -> None:
 class NPZFluorescenceFileWriter(FluorescenceFileWriter):
     def write(self, file_path: Path, dataset: FluorescenceDataset) -> None:
         element_maps = {emap.name: emap.counts_per_second for emap in dataset.element_maps}
-        numpy.savez(file_path, **element_maps)
+        numpy.savez_compressed(file_path, **element_maps)
 
 
 def register_plugins(registry: PluginRegistry) -> None: