Add jupyter notebook execution as CI test

.github/workflows/ci.yml

@@ -76,6 +76,51 @@ jobs:
       run: |
         ./bin/install.sh
 
+  Notebook:
+    runs-on: ubuntu-latest
+    needs: [Run_Installer, Download_Data]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Restore conda environment
+      id: restore-conda
+      uses: actions/cache@v4
+      env:
+        cache-name: cache-conda-environment
+      with:
+        path: ./conda
+        key: ubuntu-latest-conda-${{ hashFiles('bin/install.sh','bin/environment*.yml' ) }}
+        fail-on-cache-miss: true
+
+    - name: Restore example data
+      uses: actions/cache@v4
+      with:
+        path: |
+          example.tar.gz
+          example-cmb-lensing.tar.gz
+        key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
+        fail-on-cache-miss: true
+
+    - name: Extract test data
+      run: |
+        tar -zxvf example.tar.gz
+
+    - name: Execute notebook
+      run: |
+        source conda/bin/activate
+        jupyter execute notebooks/Welcome\ to\ TXPipe.ipynb
+
+    - name: Show logs
+      if: always()
+      run: |
+        # always run
+        cat data/example/logs_metadetect/*
+
+
   Unit_Tests:
     runs-on: ${{ matrix.os }}
     needs: Run_Installer

bin/environment-local.yml

@@ -42,7 +42,7 @@ dependencies:
     - dask-mpi==2022.4.0
     - git+https://github.com/LSSTDESC/CLMM.git@1.14.6
     - parallel-statistics==0.13
-    - sacc[all]==2.1.*
+    - sacc[all]==2.4.*
     - jax==0.7.*
     - jaxlib==0.7.*
     - glass==2023.7

bin/environment-perlmutter.yml

@@ -51,7 +51,7 @@ dependencies:
     - dask-mpi==2022.4.0
     - git+https://github.com/LSSTDESC/CLMM.git@1.14.6
     - parallel-statistics==0.13
-    - sacc[all]==2.1.*
+    - sacc[all]==2.4.*
     - jax==0.7.2
     - jaxlib==0.7.2
     - glass==2023.7

txpipe/covariance.py

@@ -687,7 +687,7 @@ def run(self):
         # Load masks
         # For clustering, we follow twopoint_fourier.py:214-219
         with self.open_input("mask", wrapper=True) as f:
-            mask = f.read_map("mask")
+            mask = f.read_map_healpix("mask")
             mask[mask == healpy.UNSEEN] = 0.0
             if self.rank == 0:
                 print("Loaded mask")
@@ -697,8 +697,11 @@ def run(self):
 
         with self.open_input("density_maps", wrapper=True) as f:
             nbin_lens = f.file["maps"].attrs["nbin_lens"]
-            d_maps = [f.read_map(f"delta_{b}") for b in range(nbin_lens)]
+            d_maps = [f.read_map_healpix(f"delta_{b}") for b in range(nbin_lens)]
             print(f"Loaded {nbin_lens} overdensity maps")
+            dmap_nside = healpy.npix2nside(d_maps[0].size)
+
+        mask = healpy.ud_grade(mask, nside_out=dmap_nside, order_in="nest", order_out="nest")
 
         # twopoint_fourier.py:219
         for d in d_maps:
@@ -708,7 +711,7 @@ def run(self):
         with self.open_input("source_maps", wrapper=True) as f:
             lensing_weights = []
             for b in range(nbin_source):
-                lw = f.read_map(f"lensing_weight_{b}")
+                lw = f.read_map_healpix(f"lensing_weight_{b}")
                 lw[lw == healpy.UNSEEN] = 0.0
                 lensing_weights.append(lw)
 
@@ -718,7 +721,7 @@ def run(self):
         # Following twopoint_fourier.py:197 all clustering maps use this mask
         masks = {f"lens_{i}": mask for i in range(nbin_lens)}
         masks.update({f"source_{i}": lensing_weights[i] for i in range(nbin_source)})
-        masks_names = {f"lens_{i}": "mask_lens" for i in range(nbin_lens)}
+        masks_names = {f"lens_{i}": f"mask_lens_{i}" for i in range(nbin_lens)}
         masks_names.update({f"source_{i}": f"mask_source_{i}" for i in range(nbin_source)})
 
         tjp_config[f"mask_file"] = masks

txpipe/maps.py

@@ -411,7 +411,7 @@ class TXDensityMaps(PipelineStage):
     outputs = [
         ("density_maps", MapsFile),
     ]
-    config_options = {"mask_threshold": StageParameter(float, 0.0, msg="Threshold for masking pixels")}
+    config_options = {"mask_threshold": StageParameter(float, 1e-14, msg="Threshold for masking pixels")}
 
     def run(self):
         import healpy

txpipe/metadata.py

@@ -114,6 +114,7 @@ def copy_source_metadata(self, meta_file, metadata, area, area_sq_arcmin):
             metadata["n_eff_2d"] = n_eff_2d.tolist()
             metadata["source_density_2d"] = source_density_2d.tolist()
             metadata["source_density"] = source_density.tolist()
+            metadata["nbin_source"] = len(source_density)
 
             copy_attrs(shear_tomo_file, "tomography", "tracers", meta_file, metadata)
 
@@ -140,6 +141,7 @@ def copy_lens_metadata(self, meta_file, metadata, area, area_sq_arcmin):
             meta_file["tracers"].attrs["density_unit"] = "arcmin^{-2}"
             metadata["lens_density"] = lens_density.tolist()
             metadata["lens_density_2d"] = lens_density_2d.tolist()
+            metadata["nbin_lens"] = len(lens_density)
 
             copy_attrs(lens_tomo_file, "tomography", "tracers", meta_file, metadata)
 

txpipe/twopoint_fourier.py

@@ -815,15 +815,14 @@ def save_power_spectra(self, tracer_sacc, nbin_source, nbin_lens):
             # the coupled noise is constant
             tr = S.tracers[tracer1]
             if (tracer1 == tracer2) and ("n_ell_coupled" not in tr.metadata):
-                if self.config["analytic_noise"] is False:
-                    # If computed through simulations, it might be better to
-                    # take the mean since, for now, only a float can be passed
-                    i = 0 if "lens" in tracer1 else 2
-                    tr.metadata["n_ell_coupled"] = np.mean(d.noise_coupled)
-                else:
+                if self.config["analytic_noise"]:
                     # Save the last element because the first one is zero for
                     # shear
                     tr.metadata["n_ell_coupled"] = d.noise_coupled[-1]
+                else:
+                    # If computed through simulations, it might be better to
+                    # take the mean since, for now, only a float can be passed
+                    tr.metadata["n_ell_coupled"] = np.mean(d.noise_coupled)
 
                 if self.config["gaussian_sims_factor"] != [1.0] and "lens" in tracer1:
                     print(tracer1)

txpipe/utils/nmt_utils.py

@@ -84,6 +84,7 @@ def choose_ell_bins(**config):
         The resulting NmtBin object.
     """
     from pymaster import NmtBin
+    config = {key.removeprefix("binning/"): val for (key, val) in config.items()}
 
     if "ell_edges" in config:
         ell_edges = config["ell_edges"]