Updates for PyPi upload and improve app memory usage

.github/workflows/pythonpublish.yml

@@ -1,6 +1,4 @@
-# This workflows will upload a Python Package using Twine when a release is created.
-
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+# This workflows will upload a Python Package to PyPi
 
 name: Upload Python Package
 
@@ -13,22 +11,25 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    permissions:
+      id-token: write  # required for PyPI Trusted Publisher (OIDC)
+
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        sudo apt-get update 
-        sudo apt-get install gfortran build-essential git f2c pkg-config libhdf5-dev -y
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist
-        twine upload dist/*
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install gfortran build-essential git f2c pkg-config libhdf5-dev -y
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

app/Dockerfile

@@ -18,4 +18,10 @@ RUN --mount=type=bind,source=.,target=/tmp/build \
     if [ -f /tmp/build/data/gedm_dist_maps.hkl ]; then cp /tmp/build/data/gedm_dist_maps.hkl /app/data/gedm_dist_maps.hkl; else wget -O /app/data/gedm_dist_maps.hkl "https://zenodo.org/records/18779007/files/gedm_dist_maps.hkl?download=1"; fi || true
 
 EXPOSE 8050/tcp
-CMD ["gunicorn", "-w", "8", "-b", "0.0.0.0:8050", "app:server"]
+# --preload: load the app once BEFORE forking workers so skymap data
+#   lives in shared copy-on-write memory instead of being duplicated.
+# -w 2: two workers is plenty for a Dash app; 8 workers each loading
+#   ~60 MB of data was consuming ~500 MB+ of RAM unnecessarily.
+# --timeout 120: data loading + first callback can take a while on cold start.
+# --access-logfile -: emit HTTP request logs to stdout for observability.
+CMD ["gunicorn", "--preload", "-w", "2", "--timeout", "120", "--access-logfile", "-", "-b", "0.0.0.0:8050", "app:server"]

app/README.md

@@ -1,12 +1,68 @@
-Install via
+# PyGEDM Web App
 
+A [Plotly Dash](https://dash.plotly.com/) web interface for [PyGEDM](https://github.com/FRBs/pygedm), providing interactive access to Galactic electron density models: **NE2001**, **NE2025**, and **YMW16**.
+
+## Features
+
+- Convert between dispersion measure (DM) and distance for any sky position
+- Interactive all-sky DM maps with distance slider
+- Supports Galactic (gl, gb) and celestial (RA, DEC) coordinates
+- Compare all three models side-by-side
+
+## Running with Docker Compose (recommended)
+
+```bash
+docker compose up --build
+```
+
+This will build the image and start the app at [localhost:8050](http://localhost:8050).
+
+To run in the background:
+
+```bash
+docker compose up -d --build
+```
+
+To stop:
+
+```bash
+docker compose down
 ```
+
+### Data file
+
+The app requires `data/gedm_dist_maps.hkl`. If it is present in `app/data/` at build time it will be copied in; otherwise the Dockerfile downloads it automatically from [Zenodo](https://zenodo.org/records/18779007).
+
+The `docker-compose.yml` mounts `./data` into the container at runtime (read-only), so you can replace the data file without rebuilding the image — just restart the container:
+
+```bash
+docker compose restart
+```
+
+## Running with Docker (manual)
+
+```bash
 docker build --tag pygedm_app .
-docker run -p 8050:8050 pygedm_app
+docker run -p 8050:8050 -v "$(pwd)/data:/app/data:ro" pygedm_app
 ```
 
-This will start a web app running on [localhost:8050](http://localhost:8050), which you can access via your browser.
+## Logs
+
+All startup and request activity is written to stdout, visible via:
+
+```bash
+docker compose logs -f
+```
+
+Key log messages to look for:
+
+| Message | Meaning |
+|---|---|
+| `Loading skymap data from …` | Data file read starting |
+| `Skymap data loaded in X.XXs (XX.X MB …)` | Successful load with timing |
+| `FATAL – could not initialise skymap data` | File missing or corrupt — check `data/` |
+| `HDF5 data group keys: …` | Lists actual keys found in the file |
 
 ## Screenshots
 
-![PyGEDM App Screenshot](assets/screenshot_v4.0.0.jpg)
+![PyGEDM App Screenshot](assets/screenshot_v4.0.0.jpg)

app/app.py

@@ -1,7 +1,20 @@
+import logging
+import sys
+import time
+
+# ── Logging setup (before any other imports so all startup is captured) ──────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    stream=sys.stdout,
+)
+logger = logging.getLogger("pygedm-app")
+logger.info("Starting pygedm web application")
+
 import dash
 import dash_bootstrap_components as dbc
 from dash import dcc, html, Input, Output, State, ctx, no_update
-import hickle as hkl
+import h5py
 import numpy as np
 import plotly.express as px
 import plotly.graph_objects as pgo
@@ -11,40 +24,82 @@
 
 import pygedm
 
-# Load skymap data
-dskymap = hkl.load('data/gedm_dist_maps.hkl')
-skymap_dist = dskymap["dist"]
-
-skymap_data_ne = xr.DataArray(
-    dskymap["ne2001"][:, ::2, ::2],
-    dims=("distance_kpc", "gb", "gl"),
-    coords={
-        "distance_kpc": skymap_dist,
-        "gl": dskymap["gl"][::2],
-        "gb": dskymap["gb"][::2],
-    },
-    attrs={"units": "DM pc/cm3"},
-)
-skymap_data_ne25 = xr.DataArray(
-    dskymap["ne2025"][:, ::2, ::2],
-    dims=("distance_kpc", "gb", "gl"),
-    coords={
-        "distance_kpc": skymap_dist,
-        "gl": dskymap["gl"][::2],
-        "gb": dskymap["gb"][::2],
-    },
-    attrs={"units": "DM pc/cm3"},
-)
-skymap_data_ymw = xr.DataArray(
-    dskymap["ymw16"][:, ::2, ::2],
-    dims=("distance_kpc", "gb", "gl"),
-    coords={
-        "distance_kpc": skymap_dist,
-        "gl": dskymap["gl"][::2],
-        "gb": dskymap["gb"][::2],
-    },
-    attrs={"units": "DM pc/cm3"},
-)
+logger.info("All imports completed")
+
+# ── Data loading ─────────────────────────────────────────────────────────────
+DATA_PATH = "data/gedm_dist_maps.hkl"
+
+
+def load_skymap_data(path):
+    """Load skymap data from HKL file using h5py directly.
+    """
+    logger.info("Loading skymap data from %s", path)
+    t0 = time.time()
+
+    try:
+        with h5py.File(path, "r") as h:
+            if "data" not in h:
+                logger.error("HDF5 file missing 'data' group. Top-level keys: %s", list(h.keys()))
+                raise KeyError("Expected 'data' group in HDF5 file")
+
+            grp = h["data"]
+            logger.info("HDF5 data group keys: %s", list(grp.keys()))
+
+            # hickle wraps dict string keys in quotes: '"keyname"'
+            dist = grp['"dist"'][()]
+            gl = grp['"gl"'][()][::2]
+            gb = grp['"gb"'][()][::2]
+
+            # Read and downsample in one step – never holds full-res in memory
+            ne2001 = grp['"ne2001"'][:, ::2, ::2]
+            ne2025 = grp['"ne2025"'][:, ::2, ::2]
+            ymw16 = grp['"ymw16"'][:, ::2, ::2]
+
+            logger.info(
+                "Loaded arrays – ne2001: %s %s, ne2025: %s %s, ymw16: %s %s",
+                ne2001.shape, ne2001.dtype,
+                ne2025.shape, ne2025.dtype,
+                ymw16.shape, ymw16.dtype,
+            )
+
+    except FileNotFoundError:
+        logger.critical("Data file not found: %s", path)
+        raise
+    except KeyError as exc:
+        logger.critical("Missing expected key in HDF5 file: %s", exc)
+        raise
+    except Exception:
+        logger.critical("Failed to load skymap data", exc_info=True)
+        raise
+
+    nbytes = sum(a.nbytes for a in (ne2001, ne2025, ymw16, dist, gl, gb))
+    elapsed = time.time() - t0
+    logger.info("Skymap data loaded in %.2fs (%.1f MB in arrays)", elapsed, nbytes / 1024**2)
+
+    return dist, gl, gb, ne2001, ne2025, ymw16
+
+
+def _build_xarray(data, dist, gl, gb):
+    """Wrap a numpy array as an xarray DataArray."""
+    return xr.DataArray(
+        data,
+        dims=("distance_kpc", "gb", "gl"),
+        coords={"distance_kpc": dist, "gl": gl, "gb": gb},
+        attrs={"units": "DM pc/cm3"},
+    )
+
+
+try:
+    _dist, _gl, _gb, _ne2001, _ne2025, _ymw16 = load_skymap_data(DATA_PATH)
+    skymap_dist = _dist
+    skymap_data_ne = _build_xarray(_ne2001, _dist, _gl, _gb)
+    skymap_data_ne25 = _build_xarray(_ne2025, _dist, _gl, _gb)
+    skymap_data_ymw = _build_xarray(_ymw16, _dist, _gl, _gb)
+    del _ne2001, _ne2025, _ymw16, _dist, _gl, _gb  # free raw arrays
+    logger.info("xarray DataArrays built successfully")
+except Exception:
+    logger.critical("FATAL – could not initialise skymap data, exiting", exc_info=True)
+    sys.exit(1)
 
 # APP SETUP
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.SPACELAB])
@@ -206,7 +261,7 @@ def callback(n_clicks, skymap_apply_clicks, relayout_data, model, colorscale, dm
             sc = SkyCoord(0 * u.deg, 0 * u.deg, frame="galactic")
             coord_error = True
 
-    print(sc.galactic.l, sc.galactic.b, dmord, f)
+    logger.info("Calculating: gl=%s, gb=%s, dmord=%s, func=%s", sc.galactic.l, sc.galactic.b, dmord, f.__name__)
     dout = f(sc.galactic.l, sc.galactic.b, dmord, method=model, nu=nu)
     dout_ne = f(sc.galactic.l, sc.galactic.b, dmord, method="ne2001", nu=nu)
     dout_ne25 = f(sc.galactic.l, sc.galactic.b, dmord, method="ne2025", nu=nu)
@@ -253,7 +308,7 @@ def callback(n_clicks, skymap_apply_clicks, relayout_data, model, colorscale, dm
         skymap_data = skymap_data_ymw
         skymap_model_label = model
 
-    print(skymap_data.shape)
+    logger.debug("Selected skymap model=%s, shape=%s", skymap_model_label, skymap_data.shape)
 
     # Determine DM min/max for colorscale
     # Only reset if distance slider changed
@@ -434,31 +489,31 @@ def _to_float(val):
     ]
 
     about_refs = [
-        html.H4("PyGEDM"),
+        html.H5("PyGEDM"),
         html.P("Price, D. C., Flynn, C., and Deller, A."),
         html.P([
             html.A(
                 "A comparison of Galactic electron density models using PyGEDM",
                 href="https://scixplorer.org/abs/2021PASA...38...38P/abstract",
             )
         ]),
-        html.H4("NE2001"),
+        html.H5("NE2001"),
         html.P("Cordes, J. M., & Lazio, T. J. W. (2002)"),
         html.P([
             html.A(
                 "NE2001.I. A New Model for the Galactic Distribution of Free Electrons and its Fluctuations, arXiv e-prints, astro-ph/0207156.",
                 href="https://ui.adsabs.harvard.edu/abs/2002astro.ph..7156C/abstract"
             )
         ]),
-        html.H4("NE2025"),
+        html.H5("NE2025"),
         html.P("Ocker, S.K. and Cordes, J.M. (2026)"),
         html.P([
             html.A(
                 "NE2025: An Updated Electron Density Model for the Galactic Interstellar Medium, arXiv e-prints, astro-ph/2602.11838.",
                 href="https://ui.adsabs.harvard.edu/abs/2026arXiv260211838O/abstract"
             )
         ]),
-        html.H4("YMW16"),
+        html.H5("YMW16"),
         html.P("Yao, J. M., Manchester, R. N., & Wang, N. (2017)"),
         html.P([
             html.A(
@@ -705,8 +760,12 @@ def _to_float(val):
                         ])
                     ], width=3),
                     dbc.Col([
-                        html.H5("References", className="mt-2"),
-                        html.Div(id="about-refs"),
+                        dbc.Card([
+                            dbc.CardBody([
+                                html.H4("References", className="card-title"),
+                                html.Div(id="about-refs"),
+                            ])
+                        ])
                     ], width=9),
                 ], className="mt-3"),
             ]),

app/docker-compose.yml

@@ -0,0 +1,26 @@
+services:
+  pygedm:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: pygedm-app
+    container_name: pygedm
+    ports:
+      - "8050:8050"
+    # Mount the data directory so the HKL file can be updated without
+    # rebuilding the image. Remove this volume if you prefer the file
+    # to be baked in at build time.
+    volumes:
+      - ./data:/app/data:ro
+    restart: unless-stopped
+    # Limit memory to prevent the container from consuming excessive RAM
+    # on the host. Adjust based on available resources.
+    mem_limit: 512m
+    environment:
+      - PYTHONUNBUFFERED=1
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8050')"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s

app/requirements.txt

@@ -12,6 +12,5 @@ gunicorn
 Pillow
 requests
 h5py
-hickle
 numba
 git+https://github.com/telegraphic/mwprop