From 0fadf34cd3ef040b36f9b2c652f09db62e8e85c4 Mon Sep 17 00:00:00 2001 From: Steve Henke Date: Fri, 9 May 2025 11:20:08 -0500 Subject: [PATCH 1/5] fix offset when calling pty-chi --- src/ptychodus/api/parametric.py | 3 +++ src/ptychodus/api/probe.py | 13 +++++++++++-- src/ptychodus/model/ptychi/helper.py | 6 ++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/ptychodus/api/parametric.py b/src/ptychodus/api/parametric.py index 5f7bbba8..8368a964 100644 --- a/src/ptychodus/api/parametric.py +++ b/src/ptychodus/api/parametric.py @@ -211,6 +211,9 @@ def get_value(self) -> float: return value + def set_value(self, value: T, *, notify: bool = True) -> None: + super().set_value(float(value), notify=notify) + def set_value_from_string(self, value: str) -> None: self.set_value(float(value)) diff --git a/src/ptychodus/api/probe.py b/src/ptychodus/api/probe.py index a2ff8785..ff823eec 100644 --- a/src/ptychodus/api/probe.py +++ b/src/ptychodus/api/probe.py @@ -169,10 +169,19 @@ def __init__( self._opr_weights = None elif numpy.issubdtype(opr_weights.dtype, numpy.floating): if opr_weights.ndim == 2: - if opr_weights.shape[1] == self._array.shape[0]: + num_weights_actual = opr_weights.shape[1] + num_weights_expected = self._array.shape[0] + + if num_weights_actual == num_weights_expected: self._opr_weights = opr_weights else: - raise ValueError('opr_weights do not match the number of coherent probe modes') + raise ValueError( + ( + 'inconsistent number of opr weights!' + f' actual={num_weights_actual}' + f' expected={num_weights_expected}' + ) + ) else: raise ValueError('opr_weights must be 2-dimensional ndarray') else: diff --git a/src/ptychodus/model/ptychi/helper.py b/src/ptychodus/model/ptychi/helper.py index dd4776f8..0add573d 100644 --- a/src/ptychodus/model/ptychi/helper.py +++ b/src/ptychodus/model/ptychi/helper.py @@ -697,16 +697,14 @@ def create_product( corrected_scan_points: list[ScanPoint] = list() object_geometry = object_in.get_geometry() - rx_px = object_geometry.width_px / 2 - ry_px = object_geometry.height_px / 2 for uncorrected_point, pos_x_px, pos_y_px in zip( product.positions, position_x_px, position_y_px ): object_point = ObjectPoint( index=uncorrected_point.index, - position_x_px=float(pos_x_px + rx_px), - position_y_px=float(pos_y_px + ry_px), + position_x_px=float(pos_x_px), + position_y_px=float(pos_y_px), ) scan_point = object_geometry.map_object_point_to_scan_point(object_point) corrected_scan_points.append(scan_point) From e520e3cd0ae834090aaf5b255321bb91d702efd1 Mon Sep 17 00:00:00 2001 From: Steve Henke Date: Mon, 12 May 2025 15:31:51 -0500 Subject: [PATCH 2/5] additional workflow hooks --- src/ptychodus/api/parametric.py | 2 +- src/ptychodus/api/workflow.py | 8 ++++++++ src/ptychodus/model/reconstructor/api.py | 3 +++ src/ptychodus/model/workflow/api.py | 6 ++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ptychodus/api/parametric.py b/src/ptychodus/api/parametric.py index 8368a964..b39e1848 100644 --- a/src/ptychodus/api/parametric.py +++ b/src/ptychodus/api/parametric.py @@ -211,7 +211,7 @@ def get_value(self) -> float: return value - def set_value(self, value: T, *, notify: bool = True) -> None: + def set_value(self, value: float, *, notify: bool = True) -> None: super().set_value(float(value), notify=notify) def set_value_from_string(self, value: str) -> None: diff --git a/src/ptychodus/api/workflow.py b/src/ptychodus/api/workflow.py index 39de93b7..bb1a9bef 100644 --- a/src/ptychodus/api/workflow.py +++ b/src/ptychodus/api/workflow.py @@ -52,6 +52,10 @@ def reconstruct_remote(self) -> None: def save_product(self, file_path: Path, *, file_type: str | None = None) -> None: pass + @abstractmethod + def export_training_data(self, file_path: Path) -> None: + pass + class WorkflowAPI(ABC): @abstractmethod @@ -103,6 +107,10 @@ def save_settings( ) -> None: pass + @abstractmethod + def set_reconstructor(self, reconstructor_name: str) -> None: + pass + class FileBasedWorkflow(ABC): @property diff --git a/src/ptychodus/model/reconstructor/api.py b/src/ptychodus/model/reconstructor/api.py index c2f0880c..26159d48 100644 --- a/src/ptychodus/model/reconstructor/api.py +++ b/src/ptychodus/model/reconstructor/api.py @@ -155,3 +155,6 @@ def train(self, data_path: Path) -> TrainOutput: logger.warning('Reconstructor is not trainable!') return result + + def set_reconstructor(self, name: str) -> None: + self._reconstructor_chooser.set_current_plugin(name) diff --git a/src/ptychodus/model/workflow/api.py b/src/ptychodus/model/workflow/api.py index f3a42782..adcb5293 100644 --- a/src/ptychodus/model/workflow/api.py +++ b/src/ptychodus/model/workflow/api.py @@ -90,6 +90,9 @@ def reconstruct_remote(self) -> None: def save_product(self, file_path: Path, *, file_type: str | None = None) -> None: self._product_api.save_product(self._product_index, file_path, file_type=file_type) + def export_training_data(self, file_path: Path) -> None: + self._reconstructor_api.export_training_data(file_path, self._product_index) + class ConcreteWorkflowAPI(WorkflowAPI): def __init__( @@ -176,3 +179,6 @@ def save_settings( self, file_path: Path, change_path_prefix: PathPrefixChange | None = None ) -> None: self._settings_registry.save_settings(file_path, change_path_prefix) + + def set_reconstructor(self, reconstructor_name: str) -> None: + self._reconstructor_api.set_reconstructor(reconstructor_name) From 62ba22be6ed1bc4186ebcff18f07b2a43145a61e Mon Sep 17 00:00:00 2001 From: Steve Henke Date: Mon, 12 May 2025 16:26:37 -0500 Subject: [PATCH 3/5] add workflow queue blocking --- src/ptychodus/api/object.py | 3 +++ src/ptychodus/api/probe.py | 3 +++ src/ptychodus/api/scan.py | 3 +++ src/ptychodus/api/workflow.py | 2 +- src/ptychodus/model/patterns/core.py | 5 +++-- src/ptychodus/model/reconstructor/api.py | 3 ++- src/ptychodus/model/workflow/api.py | 6 ++++-- src/ptychodus/plugins/npz_diffraction_file.py | 2 +- src/ptychodus/plugins/xrf_maps_file.py | 2 +- 9 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/ptychodus/api/object.py b/src/ptychodus/api/object.py index 617da303..43c128a8 100644 --- a/src/ptychodus/api/object.py +++ b/src/ptychodus/api/object.py @@ -203,6 +203,9 @@ def layer_spacing_m(self) -> Sequence[float]: def get_total_thickness_m(self) -> float: return sum(self._layer_spacing_m) + def __repr__(self) -> str: + return f'{self._array.dtype}{self._array.shape}' + class ObjectFileReader(ABC): @abstractmethod diff --git a/src/ptychodus/api/probe.py b/src/ptychodus/api/probe.py index ff823eec..5de60d73 100644 --- a/src/ptychodus/api/probe.py +++ b/src/ptychodus/api/probe.py @@ -276,6 +276,9 @@ def get_geometry(self) -> ProbeGeometry: def __len__(self) -> int: return 1 if self._opr_weights is None else self._opr_weights.shape[0] + def __repr__(self) -> str: + return f'{self._array.dtype}{self._array.shape}' + class ProbeFileReader(ABC): @abstractmethod diff --git a/src/ptychodus/api/scan.py b/src/ptychodus/api/scan.py index e8e64bf0..ed047013 100644 --- a/src/ptychodus/api/scan.py +++ b/src/ptychodus/api/scan.py @@ -87,6 +87,9 @@ def __len__(self) -> int: def nbytes(self) -> int: return self._coordinates_m.nbytes + def __repr__(self) -> str: + return f'{self._coordinates_m.dtype}{self._coordinates_m.shape}' + class ScanPointParseError(Exception): """raised when the scan file cannot be parsed""" diff --git a/src/ptychodus/api/workflow.py b/src/ptychodus/api/workflow.py index bb1a9bef..738efe25 100644 --- a/src/ptychodus/api/workflow.py +++ b/src/ptychodus/api/workflow.py @@ -41,7 +41,7 @@ def build_object( pass @abstractmethod - def reconstruct_local(self) -> WorkflowProductAPI: + def reconstruct_local(self, block: bool = False) -> WorkflowProductAPI: pass @abstractmethod diff --git a/src/ptychodus/model/patterns/core.py b/src/ptychodus/model/patterns/core.py index 0ef5e6c2..41bdd399 100644 --- a/src/ptychodus/model/patterns/core.py +++ b/src/ptychodus/model/patterns/core.py @@ -47,7 +47,7 @@ def start(self) -> None: pass def stop(self) -> None: - self.dataset.finish_loading(block=False) + self.patterns_api.finish_assembling_diffraction_patterns(block=False) def _update(self, observable: Observable) -> None: if observable is self._reinit_observable: @@ -55,4 +55,5 @@ def _update(self, observable: Observable) -> None: file_path=self.pattern_settings.file_path.get_value(), file_type=self.pattern_settings.file_type.get_value(), ) - self.dataset.start_loading() + self.patterns_api.start_assembling_diffraction_patterns() + self.patterns_api.finish_assembling_diffraction_patterns(block=True) diff --git a/src/ptychodus/model/reconstructor/api.py b/src/ptychodus/model/reconstructor/api.py index 26159d48..356ac0cc 100644 --- a/src/ptychodus/model/reconstructor/api.py +++ b/src/ptychodus/model/reconstructor/api.py @@ -156,5 +156,6 @@ def train(self, data_path: Path) -> TrainOutput: return result - def set_reconstructor(self, name: str) -> None: + def set_reconstructor(self, name: str) -> str: self._reconstructor_chooser.set_current_plugin(name) + return self._reconstructor_chooser.get_current_plugin().simple_name diff --git a/src/ptychodus/model/workflow/api.py b/src/ptychodus/model/workflow/api.py index adcb5293..c8ee130b 100644 --- a/src/ptychodus/model/workflow/api.py +++ b/src/ptychodus/model/workflow/api.py @@ -69,9 +69,10 @@ def build_object( else: self._object_api.build_object(self._product_index, builder_name, builder_parameters) - def reconstruct_local(self) -> WorkflowProductAPI: + def reconstruct_local(self, block: bool = False) -> WorkflowProductAPI: logger.debug(f'Reconstruct: index={self._product_index}') output_product_index = self._reconstructor_api.reconstruct(self._product_index) + self._reconstructor_api.process_results(block=block) return ConcreteWorkflowProductAPI( self._product_api, @@ -181,4 +182,5 @@ def save_settings( self._settings_registry.save_settings(file_path, change_path_prefix) def set_reconstructor(self, reconstructor_name: str) -> None: - self._reconstructor_api.set_reconstructor(reconstructor_name) + reconstructor = self._reconstructor_api.set_reconstructor(reconstructor_name) + logger.debug(f'{reconstructor=}') diff --git a/src/ptychodus/plugins/npz_diffraction_file.py b/src/ptychodus/plugins/npz_diffraction_file.py index 111e3575..2337e087 100644 --- a/src/ptychodus/plugins/npz_diffraction_file.py +++ b/src/ptychodus/plugins/npz_diffraction_file.py @@ -79,7 +79,7 @@ def write(self, file_path: Path, dataset: DiffractionDataset) -> None: self.INDEXES: numpy.concatenate([array.get_indexes() for array in dataset]), self.PATTERNS: numpy.concatenate([array.get_data() for array in dataset]), } - numpy.savez(file_path, **contents) + numpy.savez_compressed(file_path, **contents) def register_plugins(registry: PluginRegistry) -> None: diff --git a/src/ptychodus/plugins/xrf_maps_file.py b/src/ptychodus/plugins/xrf_maps_file.py index f78ce7f0..c54461c7 100644 --- a/src/ptychodus/plugins/xrf_maps_file.py +++ b/src/ptychodus/plugins/xrf_maps_file.py @@ -85,7 +85,7 @@ def write(self, file_path: Path, dataset: FluorescenceDataset) -> None: class NPZFluorescenceFileWriter(FluorescenceFileWriter): def write(self, file_path: Path, dataset: FluorescenceDataset) -> None: element_maps = {emap.name: emap.counts_per_second for emap in dataset.element_maps} - numpy.savez(file_path, **element_maps) + numpy.savez_compressed(file_path, **element_maps) def register_plugins(registry: PluginRegistry) -> None: From 134de46bfc20aec8ff5e602ab4811c0d42ea0fa7 Mon Sep 17 00:00:00 2001 From: Steve Henke Date: Tue, 13 May 2025 12:10:31 -0500 Subject: [PATCH 4/5] fix reconstructor blocking --- src/ptychodus/api/workflow.py | 4 ++++ src/ptychodus/model/core.py | 28 ++++------------------ src/ptychodus/model/reconstructor/queue.py | 14 +++++++++++ src/ptychodus/model/workflow/api.py | 10 +++++++- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/ptychodus/api/workflow.py b/src/ptychodus/api/workflow.py index 738efe25..9e60f283 100644 --- a/src/ptychodus/api/workflow.py +++ b/src/ptychodus/api/workflow.py @@ -10,6 +10,10 @@ class WorkflowProductAPI(ABC): + @abstractmethod + def get_product_index(self) -> int: + pass + @abstractmethod def open_scan(self, file_path: Path, *, file_type: str | None = None) -> None: pass diff --git a/src/ptychodus/model/core.py b/src/ptychodus/model/core.py index f4b33714..3f97825d 100644 --- a/src/ptychodus/model/core.py +++ b/src/ptychodus/model/core.py @@ -231,7 +231,6 @@ def batch_mode_execute( input_path: Path, output_path: Path, *, - product_file_type: str = 'NPZ', fluorescence_input_file_path: Path | None = None, fluorescence_output_file_path: Path | None = None, ) -> int: @@ -241,39 +240,20 @@ def batch_mode_execute( self.reconstructor.reconstructor_api.save_model(output_path) return output.result - input_product_index = self.product.product_api.open_product( - input_path, file_type=product_file_type - ) - - if input_product_index < 0: - logger.error(f'Failed to open product "{input_path}"!') - return -1 - if action.lower() == 'reconstruct': - logger.info('Reconstructing...') - output_product_index = self.reconstructor.reconstructor_api.reconstruct( - input_product_index - ) - self.reconstructor.reconstructor_api.process_results(block=True) - logger.info('Reconstruction complete.') - - self.product.product_api.save_product( - output_product_index, output_path, file_type=product_file_type - ) + input_product_api = self.workflow.workflow_api.open_product(input_path) + output_product_api = input_product_api.reconstruct_local(block=True) + output_product_api.save_product(output_path) if ( fluorescence_input_file_path is not None and fluorescence_output_file_path is not None ): self.fluorescence_core.enhance_fluorescence( - output_product_index, + output_product_api.get_product_index(), fluorescence_input_file_path, fluorescence_output_file_path, ) - elif action.lower() == 'prepare_training_data': - self.reconstructor.reconstructor_api.export_training_data( - output_path, input_product_index - ) else: logger.error(f'Unknown batch mode action "{action}"!') return -1 diff --git a/src/ptychodus/model/reconstructor/queue.py b/src/ptychodus/model/reconstructor/queue.py index 8b6fcf47..387366d8 100644 --- a/src/ptychodus/model/reconstructor/queue.py +++ b/src/ptychodus/model/reconstructor/queue.py @@ -83,11 +83,15 @@ def _reconstruct(self) -> None: input_task = self._input_queue.get(block=True, timeout=1) try: + logger.debug('Reconstructing...') output_task = input_task.execute() + logger.debug('Reconstruction finished.') except Exception: logger.exception('Reconstructor error!') else: + logger.debug('Adding reconstruction result to output queue...') self._output_queue.put(output_task) + logger.debug('Reconstruction result added to output queue.') finally: self._input_queue.task_done() except queue.Empty: @@ -102,17 +106,27 @@ def put( task = ExecuteReconstructorTask( self._data_matcher, reconstructor, product_index, index_filter ) + logger.debug('Adding reconstruction task to queue...') self._input_queue.put(task) + logger.debug('Reconstruction task added to queue.') def process_results(self, *, block: bool) -> None: while True: try: + logger.debug('Waiting for reconstruction result...') task = self._output_queue.get(block=block) + logger.debug('Reconstruction result received.') try: + logger.debug('Processing reconstruction result...') task.execute() + logger.debug('Reconstruction result processed.') finally: self._output_queue.task_done() + + if block and self._output_queue.empty(): + logger.debug('No more results to process.') + break except queue.Empty: break diff --git a/src/ptychodus/model/workflow/api.py b/src/ptychodus/model/workflow/api.py index c8ee130b..2aa04073 100644 --- a/src/ptychodus/model/workflow/api.py +++ b/src/ptychodus/model/workflow/api.py @@ -36,6 +36,9 @@ def __init__( self._executor = executor self._product_index = product_index + def get_product_index(self) -> int: + return self._product_index + def open_scan(self, file_path: Path, *, file_type: str | None = None) -> None: self._scan_api.open_scan(self._product_index, file_path, file_type=file_type) @@ -70,9 +73,10 @@ def build_object( self._object_api.build_object(self._product_index, builder_name, builder_parameters) def reconstruct_local(self, block: bool = False) -> WorkflowProductAPI: - logger.debug(f'Reconstruct: index={self._product_index}') + logger.info('Reconstructing...') output_product_index = self._reconstructor_api.reconstruct(self._product_index) self._reconstructor_api.process_results(block=block) + logger.info('Reconstruction complete.') return ConcreteWorkflowProductAPI( self._product_api, @@ -150,6 +154,10 @@ def export_assembled_patterns(self, file_path: Path) -> None: def open_product(self, file_path: Path, *, file_type: str | None = None) -> WorkflowProductAPI: product_index = self._product_api.open_product(file_path, file_type=file_type) + + if product_index < 0: + raise RuntimeError(f'Failed to open product "{file_path}"!') + return self._create_product_api(product_index) def create_product( From e7bc124f8a16a8dc8a150884d68dde1e32f34508 Mon Sep 17 00:00:00 2001 From: Steve Henke Date: Thu, 15 May 2025 09:56:00 -0500 Subject: [PATCH 5/5] add method to get product by index to workflow api --- src/ptychodus/api/workflow.py | 5 +++ src/ptychodus/model/product/item_factory.py | 4 ++- src/ptychodus/model/reconstructor/matcher.py | 2 +- src/ptychodus/model/workflow/api.py | 32 ++++++++++---------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/ptychodus/api/workflow.py b/src/ptychodus/api/workflow.py index 9e60f283..c7d7ae63 100644 --- a/src/ptychodus/api/workflow.py +++ b/src/ptychodus/api/workflow.py @@ -84,6 +84,11 @@ def export_assembled_patterns(self, file_path: Path) -> None: """export assembled patterns""" pass + @abstractmethod + def get_product(self, product_index: int) -> WorkflowProductAPI: + """returns a product by index""" + pass + @abstractmethod def open_product(self, file_path: Path, *, file_type: str | None = None) -> WorkflowProductAPI: """opens product from file""" diff --git a/src/ptychodus/model/product/item_factory.py b/src/ptychodus/model/product/item_factory.py index 0b25d1b3..2e15ab53 100644 --- a/src/ptychodus/model/product/item_factory.py +++ b/src/ptychodus/model/product/item_factory.py @@ -136,7 +136,7 @@ def create_from_settings(self) -> ProductRepositoryItem: probe_item = self._probe_item_factory.create_from_settings(geometry) object_item = self._object_item_factory.create_from_settings(geometry) - return ProductRepositoryItem( + item = ProductRepositoryItem( parent=self._repository, metadata_item=metadata_item, scan_item=scan_item, @@ -146,3 +146,5 @@ def create_from_settings(self) -> ProductRepositoryItem: validator=ProductValidator(self._dataset, scan_item, geometry, probe_item, object_item), costs=list(), ) + logger.debug(f'Created product from settings: {item.get_name()}') + return item diff --git a/src/ptychodus/model/reconstructor/matcher.py b/src/ptychodus/model/reconstructor/matcher.py index 6c974fa9..89399f83 100644 --- a/src/ptychodus/model/reconstructor/matcher.py +++ b/src/ptychodus/model/reconstructor/matcher.py @@ -53,7 +53,7 @@ def match_diffraction_patterns_with_positions( ) -> ReconstructInput: input_product_item = self._product_api.get_item(input_product_index) input_product = input_product_item.get_product() - data_indexes = self._dataset.get_assembled_indexes() + data_indexes = [int(index) for index in self._dataset.get_assembled_indexes()] scan_indexes = [ point.index for point in input_product.positions if index_filter(point.index) ] diff --git a/src/ptychodus/model/workflow/api.py b/src/ptychodus/model/workflow/api.py index 2aa04073..dfd9b1d4 100644 --- a/src/ptychodus/model/workflow/api.py +++ b/src/ptychodus/model/workflow/api.py @@ -120,20 +120,6 @@ def __init__( self._reconstructor_api = reconstructor_api self._executor = executor - def _create_product_api(self, product_index: int) -> WorkflowProductAPI: - if product_index < 0: - raise ValueError(f'Bad product index ({product_index=})!') - - return ConcreteWorkflowProductAPI( - self._product_api, - self._scan_api, - self._probe_api, - self._object_api, - self._reconstructor_api, - self._executor, - product_index, - ) - def open_patterns( self, file_path: Path, @@ -152,13 +138,27 @@ def import_assembled_patterns(self, file_path: Path) -> None: def export_assembled_patterns(self, file_path: Path) -> None: self._patterns_api.export_assembled_patterns(file_path) + def get_product(self, product_index: int) -> WorkflowProductAPI: + if product_index < 0: + raise ValueError(f'Bad product index ({product_index=})!') + + return ConcreteWorkflowProductAPI( + self._product_api, + self._scan_api, + self._probe_api, + self._object_api, + self._reconstructor_api, + self._executor, + product_index, + ) + def open_product(self, file_path: Path, *, file_type: str | None = None) -> WorkflowProductAPI: product_index = self._product_api.open_product(file_path, file_type=file_type) if product_index < 0: raise RuntimeError(f'Failed to open product "{file_path}"!') - return self._create_product_api(product_index) + return self.get_product(product_index) def create_product( self, @@ -182,7 +182,7 @@ def create_product( mass_attenuation_m2_kg=mass_attenuation_m2_kg, tomography_angle_deg=tomography_angle_deg, ) - return self._create_product_api(product_index) + return self.get_product(product_index) def save_settings( self, file_path: Path, change_path_prefix: PathPrefixChange | None = None