Use file_format and use new FileIO hyperspy metadata

doc/file_specification/index.rst

@@ -34,7 +34,7 @@ MRC (CCP-EM)
 
 The :ref:`MRC <mrc-format>` file format is a standard open file format for electron microscopy data and is
 defined by  `Cheng et al <https://doi.org/10.1016/j.jsb.2015.04.002>`_. The file format is described in
-detail following the link as well:  `MRC2014 <https://www.ccpem.ac.uk/mrc_format/mrc2014.php>`_.
+detail following the link as well:  `MRC2014 <https://www.ccpem.ac.uk/mrc-format>`_.
 
 Additionally Direct Electron saves a couple of files along with the ``.mrc`` file. In general, the file naming
 scheme is CurrentDate_MovieNumber_suffix_movie.mrc with the metadata file named CurrentDate_MovieNumber_suffix_info.txt.

doc/supported_formats/mrcz.rst

@@ -8,7 +8,7 @@ MRCZ format
    required.
 
 The ``mrcz`` format is an extension of the CCP-EM MRC2014 file format.
-`CCP-EM MRC2014 <https://www.ccpem.ac.uk/mrc_format/mrc2014.php>`_ file format.
+`CCP-EM MRC2014 <https://www.ccpem.ac.uk/mrc-format>`_ file format.
 It uses the `blosc` meta-compression library to bitshuffle and compress files in
 a blocked, multi-threaded environment. The supported data types are ``float32``,
 ``int8``, ``uint16``, ``int16`` and ``complex64``.

rsciio/emd/_api.py

@@ -25,6 +25,7 @@
 
 import json
 import logging
+from pathlib import Path
 
 import h5py
 
@@ -95,6 +96,9 @@ def _is_EMD_velox(file):
                 return True
         return False
 
+    if isinstance(file, Path):
+        file = str(file)
+
     if isinstance(file, str):
         with h5py.File(file, "r") as f:
             return _is_EMD_velox(f)
@@ -173,6 +177,9 @@ def file_reader(
     ModuleNotFoundError
         When reading spectrum image from Velox EMD file and the ``sparse`` library is missing.
     """
+    if isinstance(filename, Path):
+        filename = str(filename)
+
     file = h5py.File(filename, "r")
     dictionaries = []
     try:
@@ -230,6 +237,8 @@ def file_writer(filename, signal, chunks=None, **kwds):
         Dictionary containing metadata, which will be written as attribute
         of the root group.
     """
+    if isinstance(filename, Path):
+        filename = str(filename)
     from ._emd_ncem import EMD_NCEM
 
     EMD_NCEM().write_file(filename, signal, chunks=chunks, **kwds)

rsciio/hspy/_api.py

@@ -212,7 +212,10 @@ def file_writer(
     folder = signal["tmp_parameters"].get("original_folder", "")
     fname = signal["tmp_parameters"].get("original_filename", "")
     ext = signal["tmp_parameters"].get("original_extension", "")
-    original_path = Path(folder, f"{fname}.{ext}")
+    if ext and not ext.startswith("."):
+        # add a dot if not present
+        ext = f".{ext}"
+    original_path = Path(folder, f"{fname}{ext}")
 
     f = None
     if signal["attributes"]["_lazy"] and Path(filename).absolute() == original_path:

rsciio/tests/test_bruker.py

@@ -3,7 +3,9 @@
 
 import numpy as np
 import pytest
+from packaging.version import Version
 
+import rsciio
 from rsciio.bruker import export_metadata, file_reader
 from rsciio.utils.tests import assert_deep_almost_equal
 
@@ -147,8 +149,12 @@ def test_hyperspy_wrap():
             "FileIO": {
                 "0": {
                     "operation": "load",
+                    "folder": str(TEST_DATA_DIR),
+                    "filename": "30x30_instructively_packed_16bit_compressed",
+                    "extension": ".bcf",
                     "hyperspy_version": hs.__version__,
                     "io_plugin": "rsciio.bruker",
+                    "rosettasciio_version": rsciio.__version__,
                 }
             },
         },
@@ -207,7 +213,13 @@ def test_hyperspy_wrap():
         # original_metadata:
         omd_ref = json.load(fn)
     # delete FileIO timestamp since it's runtime dependent
-    del hype.metadata.General.FileIO.Number_0.timestamp
+    del hype.metadata.General.FileIO[0].timestamp
+    if Version(hs.__version__) < Version("2.4.0.dev64"):
+        del md_ref["General"]["FileIO"]["0"]["folder"]
+        del md_ref["General"]["FileIO"]["0"]["filename"]
+        del md_ref["General"]["FileIO"]["0"]["extension"]
+        del md_ref["General"]["FileIO"]["0"]["rosettasciio_version"]
+
     assert_deep_almost_equal(hype.metadata.as_dictionary(), md_ref)
     assert_deep_almost_equal(hype.original_metadata.as_dictionary(), omd_ref)
     assert hype.metadata.General.date == "2018-10-04"

rsciio/tests/test_delmic.py

@@ -2,13 +2,18 @@
 
 import numpy as np
 import pytest
+from packaging.version import Version
 
 hs = pytest.importorskip("hyperspy.api", reason="hyperspy not installed")
 pytest.importorskip("h5py", reason="h5py not installed")
 
 testfile_dir = (Path(__file__).parent / "data" / "delmic").resolve()
 testfile_hyperspectral_path = (testfile_dir / "test_hyperspectral.h5").resolve()
 
+argument_name = (
+    "reader" if Version(hs.__version__) < Version("2.4.0.dev33") else "file_format"
+)
+hs_load_kwargs = {argument_name: "Delmic"}
 
 ref = np.array(
     [
@@ -529,7 +534,7 @@
 
 
 def test_read():
-    s = hs.load(testfile_hyperspectral_path, reader="Delmic")
+    s = hs.load(testfile_hyperspectral_path, **hs_load_kwargs)
 
     np.testing.assert_allclose(s.axes_manager[0].scale, 0.00010759749808557962)
     np.testing.assert_allclose(s.axes_manager[0].offset, 0)

rsciio/tests/test_digitalmicrograph.py

@@ -22,7 +22,9 @@
 
 import numpy as np
 import pytest
+from packaging.version import Version
 
+import rsciio
 from rsciio.digitalmicrograph._api import (
     DigitalMicrographReader,
     ImageObject,
@@ -533,8 +535,12 @@ def test_multi_signal():
             "FileIO": {
                 "0": {
                     "operation": "load",
+                    "folder": str(DM_2D_PATH),
+                    "filename": fname.stem,
+                    "extension": ".dm3",
                     "hyperspy_version": hs.__version__,
                     "io_plugin": "rsciio.digitalmicrograph",
+                    "rosettasciio_version": rsciio.__version__,
                 }
             },
         },
@@ -580,8 +586,12 @@ def test_multi_signal():
             "FileIO": {
                 "0": {
                     "operation": "load",
+                    "folder": str(DM_2D_PATH),
+                    "filename": fname.stem,
+                    "extension": ".dm3",
                     "hyperspy_version": hs.__version__,
                     "io_plugin": "rsciio.digitalmicrograph",
+                    "rosettasciio_version": rsciio.__version__,
                 }
             },
         },
@@ -594,8 +604,14 @@ def test_multi_signal():
         },
     }
     # remove timestamps from metadata since these are runtime dependent
-    del s1.metadata.General.FileIO.Number_0.timestamp
-    del s2.metadata.General.FileIO.Number_0.timestamp
+    del s1.metadata.General.FileIO[0].timestamp
+    del s2.metadata.General.FileIO[0].timestamp
+    for md in [s1_md_truth, s2_md_truth]:
+        if Version(hs.__version__) < Version("2.4.0.dev64"):
+            del md["General"]["FileIO"]["0"]["folder"]
+            del md["General"]["FileIO"]["0"]["filename"]
+            del md["General"]["FileIO"]["0"]["extension"]
+            del md["General"]["FileIO"]["0"]["rosettasciio_version"]
 
     # make sure the metadata dictionaries are as we expect
     assert s1.metadata.as_dictionary() == s1_md_truth

rsciio/tests/test_emd_velox.py

@@ -31,7 +31,9 @@
 import numpy as np
 import pytest
 from dateutil import tz
+from packaging.version import Version
 
+import rsciio
 from rsciio.utils.tests import assert_deep_almost_equal
 from rsciio.utils.tools import dummy_context_manager
 
@@ -94,8 +96,12 @@ def test_fei_emd_image(self, lazy):
                 "FileIO": {
                     "0": {
                         "operation": "load",
+                        "folder": str(self.fei_files_path),
+                        "filename": "fei_emd_image",
+                        "extension": ".emd",
                         "hyperspy_version": hs.__version__,
                         "io_plugin": "rsciio.emd",
+                        "rosettasciio_version": rsciio.__version__,
                     }
                 },
             },
@@ -119,7 +125,13 @@ def test_fei_emd_image(self, lazy):
 
         signal = hs.load(self.fei_files_path / "fei_emd_image.emd", lazy=lazy)
         # delete timestamp from metadata since it's runtime dependent
-        del signal.metadata.General.FileIO.Number_0.timestamp
+        del signal.metadata.General.FileIO[0].timestamp
+        if Version(hs.__version__) < Version("2.4.0.dev64"):
+            del md["General"]["FileIO"]["0"]["folder"]
+            del md["General"]["FileIO"]["0"]["filename"]
+            del md["General"]["FileIO"]["0"]["extension"]
+            del md["General"]["FileIO"]["0"]["rosettasciio_version"]
+
         if lazy:
             assert signal._lazy
             signal.compute(close_file=True)

rsciio/tests/test_empad.py

@@ -35,6 +35,11 @@
 FILENAME_STACK_RAW = DATA_DIR / "series_x10.raw"
 FILENAME_MAP_RAW = DATA_DIR / "scan_x4_y4.raw"
 
+argument_name = (
+    "reader" if Version(hs.__version__) < Version("2.4.0.dev33") else "file_format"
+)
+hs_load_kwargs = {argument_name: "EMPAD"}
+
 
 def _create_raw_data(filename, shape):
     size = np.prod(shape)
@@ -61,7 +66,7 @@ def teardown_module(module):
 @pytest.mark.parametrize("lazy", (False, True))
 def test_read_stack(lazy):
     # xml file version 0.51 211118
-    s = hs.load(DATA_DIR / "stack_images.xml", lazy=lazy, reader="EMPAD")
+    s = hs.load(DATA_DIR / "stack_images.xml", lazy=lazy, **hs_load_kwargs)
     assert s.data.dtype == "float32"
     ref_data = np.arange(166400).reshape((10, 130, 128))[..., :128, :]
     np.testing.assert_allclose(s.data, ref_data.astype("float32"))
@@ -92,7 +97,7 @@ def test_read_stack(lazy):
 @pytest.mark.parametrize("lazy", (False, True))
 def test_read_map(lazy):
     # xml file version 0.51 211118
-    s = hs.load(DATA_DIR / "map4x4.xml", lazy=lazy, reader="EMPAD")
+    s = hs.load(DATA_DIR / "map4x4.xml", lazy=lazy, **hs_load_kwargs)
     assert s.data.dtype == "float32"
     ref_data = np.arange(266240).reshape((4, 4, 130, 128))[..., :128, :]
     np.testing.assert_allclose(s.data, ref_data.astype("float32"))

rsciio/tests/test_hamamatsu.py

@@ -22,6 +22,7 @@
 
 import numpy as np
 import pytest
+from packaging.version import Version
 
 hs = pytest.importorskip("hyperspy.api", reason="hyperspy not installed")
 
@@ -33,11 +34,16 @@
 testfile_shading_path = (testfile_dir / "shading_file.img").resolve()
 testfile_xaxisother_path = (testfile_dir / "xaxis_other.img").resolve()
 
+argument_name = (
+    "reader" if Version(hs.__version__) < Version("2.4.0.dev33") else "file_format"
+)
+hs_load_kwargs = {argument_name: "Hamamatsu"}
+
 
 class TestOperate:
     @classmethod
     def setup_class(cls):
-        cls.s = hs.load(testfile_operate_mode_path, reader="Hamamatsu")
+        cls.s = hs.load(testfile_operate_mode_path, **hs_load_kwargs)
 
     @classmethod
     def teardown_class(cls):
@@ -329,7 +335,7 @@ def test_lumispy(self):
 class TestFocus:
     @classmethod
     def setup_class(cls):
-        cls.s_focus = hs.load(testfile_focus_mode_path, reader="Hamamatsu")
+        cls.s_focus = hs.load(testfile_focus_mode_path, **hs_load_kwargs)
 
     @classmethod
     def teardown_class(cls):
@@ -363,7 +369,7 @@ def test_axes_focus(self):
 class TestPhotonCount:
     @classmethod
     def setup_class(cls):
-        cls.s = hs.load(testfile_photon_count_path, reader="Hamamatsu")
+        cls.s = hs.load(testfile_photon_count_path, **hs_load_kwargs)
 
     @classmethod
     def teardown_class(cls):
@@ -390,7 +396,7 @@ def test_metadata(self):
 class TestShading:
     @classmethod
     def setup_class(cls):
-        cls.s = hs.load(testfile_shading_path, reader="Hamamatsu")
+        cls.s = hs.load(testfile_shading_path, **hs_load_kwargs)
 
     @classmethod
     def teardown_class(cls):
@@ -410,7 +416,7 @@ def test_data(self):
 class TestOther:
     @classmethod
     def setup_class(cls):
-        cls.s = hs.load(testfile_xaxisother_path, reader="Hamamatsu")
+        cls.s = hs.load(testfile_xaxisother_path, **hs_load_kwargs)
 
     @classmethod
     def teardown_class(cls):