diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 55de772..85191ce 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,6 +1,4 @@ -# This workflows will upload a Python Package using Twine when a release is created. - -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries +# This workflows will upload a Python Package to PyPi name: Upload Python Package @@ -13,22 +11,25 @@ jobs: runs-on: ubuntu-latest + permissions: + id-token: write # required for PyPI Trusted Publisher (OIDC) + steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.x' - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install gfortran build-essential git f2c pkg-config libhdf5-dev -y - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist - twine upload dist/* + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install gfortran build-essential git f2c pkg-config libhdf5-dev -y + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build sdist + run: pipx run build --sdist + + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/app/Dockerfile b/app/Dockerfile index c20ea83..321d708 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -18,4 +18,10 @@ RUN --mount=type=bind,source=.,target=/tmp/build \ if [ -f /tmp/build/data/gedm_dist_maps.hkl ]; then cp /tmp/build/data/gedm_dist_maps.hkl /app/data/gedm_dist_maps.hkl; else wget -O /app/data/gedm_dist_maps.hkl "https://zenodo.org/records/18779007/files/gedm_dist_maps.hkl?download=1"; fi || true EXPOSE 8050/tcp -CMD ["gunicorn", "-w", "8", "-b", "0.0.0.0:8050", "app:server"] +# --preload: load the app once BEFORE forking workers so skymap data +# lives in shared copy-on-write memory instead of being duplicated. +# -w 2: two workers is plenty for a Dash app; 8 workers each loading +# ~60 MB of data was consuming ~500 MB+ of RAM unnecessarily. +# --timeout 120: data loading + first callback can take a while on cold start. +# --access-logfile -: emit HTTP request logs to stdout for observability. +CMD ["gunicorn", "--preload", "-w", "2", "--timeout", "120", "--access-logfile", "-", "-b", "0.0.0.0:8050", "app:server"] diff --git a/app/README.md b/app/README.md index 91b13a6..c94aac6 100644 --- a/app/README.md +++ b/app/README.md @@ -1,12 +1,68 @@ -Install via +# PyGEDM Web App +A [Plotly Dash](https://dash.plotly.com/) web interface for [PyGEDM](https://github.com/FRBs/pygedm), providing interactive access to Galactic electron density models: **NE2001**, **NE2025**, and **YMW16**. + +## Features + +- Convert between dispersion measure (DM) and distance for any sky position +- Interactive all-sky DM maps with distance slider +- Supports Galactic (gl, gb) and celestial (RA, DEC) coordinates +- Compare all three models side-by-side + +## Running with Docker Compose (recommended) + +```bash +docker compose up --build +``` + +This will build the image and start the app at [localhost:8050](http://localhost:8050). + +To run in the background: + +```bash +docker compose up -d --build +``` + +To stop: + +```bash +docker compose down ``` + +### Data file + +The app requires `data/gedm_dist_maps.hkl`. If it is present in `app/data/` at build time it will be copied in; otherwise the Dockerfile downloads it automatically from [Zenodo](https://zenodo.org/records/18779007). + +The `docker-compose.yml` mounts `./data` into the container at runtime (read-only), so you can replace the data file without rebuilding the image — just restart the container: + +```bash +docker compose restart +``` + +## Running with Docker (manual) + +```bash docker build --tag pygedm_app . -docker run -p 8050:8050 pygedm_app +docker run -p 8050:8050 -v "$(pwd)/data:/app/data:ro" pygedm_app ``` -This will start a web app running on [localhost:8050](http://localhost:8050), which you can access via your browser. +## Logs + +All startup and request activity is written to stdout, visible via: + +```bash +docker compose logs -f +``` + +Key log messages to look for: + +| Message | Meaning | +|---|---| +| `Loading skymap data from …` | Data file read starting | +| `Skymap data loaded in X.XXs (XX.X MB …)` | Successful load with timing | +| `FATAL – could not initialise skymap data` | File missing or corrupt — check `data/` | +| `HDF5 data group keys: …` | Lists actual keys found in the file | ## Screenshots -![PyGEDM App Screenshot](assets/screenshot_v4.0.0.jpg) \ No newline at end of file +![PyGEDM App Screenshot](assets/screenshot_v4.0.0.jpg) diff --git a/app/app.py b/app/app.py index 8147e50..a00c70d 100644 --- a/app/app.py +++ b/app/app.py @@ -1,7 +1,20 @@ +import logging +import sys +import time + +# ── Logging setup (before any other imports so all startup is captured) ────── +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + stream=sys.stdout, +) +logger = logging.getLogger("pygedm-app") +logger.info("Starting pygedm web application") + import dash import dash_bootstrap_components as dbc from dash import dcc, html, Input, Output, State, ctx, no_update -import hickle as hkl +import h5py import numpy as np import plotly.express as px import plotly.graph_objects as pgo @@ -11,40 +24,82 @@ import pygedm -# Load skymap data -dskymap = hkl.load('data/gedm_dist_maps.hkl') -skymap_dist = dskymap["dist"] - -skymap_data_ne = xr.DataArray( - dskymap["ne2001"][:, ::2, ::2], - dims=("distance_kpc", "gb", "gl"), - coords={ - "distance_kpc": skymap_dist, - "gl": dskymap["gl"][::2], - "gb": dskymap["gb"][::2], - }, - attrs={"units": "DM pc/cm3"}, -) -skymap_data_ne25 = xr.DataArray( - dskymap["ne2025"][:, ::2, ::2], - dims=("distance_kpc", "gb", "gl"), - coords={ - "distance_kpc": skymap_dist, - "gl": dskymap["gl"][::2], - "gb": dskymap["gb"][::2], - }, - attrs={"units": "DM pc/cm3"}, -) -skymap_data_ymw = xr.DataArray( - dskymap["ymw16"][:, ::2, ::2], - dims=("distance_kpc", "gb", "gl"), - coords={ - "distance_kpc": skymap_dist, - "gl": dskymap["gl"][::2], - "gb": dskymap["gb"][::2], - }, - attrs={"units": "DM pc/cm3"}, -) +logger.info("All imports completed") + +# ── Data loading ───────────────────────────────────────────────────────────── +DATA_PATH = "data/gedm_dist_maps.hkl" + + +def load_skymap_data(path): + """Load skymap data from HKL file using h5py directly. + """ + logger.info("Loading skymap data from %s", path) + t0 = time.time() + + try: + with h5py.File(path, "r") as h: + if "data" not in h: + logger.error("HDF5 file missing 'data' group. Top-level keys: %s", list(h.keys())) + raise KeyError("Expected 'data' group in HDF5 file") + + grp = h["data"] + logger.info("HDF5 data group keys: %s", list(grp.keys())) + + # hickle wraps dict string keys in quotes: '"keyname"' + dist = grp['"dist"'][()] + gl = grp['"gl"'][()][::2] + gb = grp['"gb"'][()][::2] + + # Read and downsample in one step – never holds full-res in memory + ne2001 = grp['"ne2001"'][:, ::2, ::2] + ne2025 = grp['"ne2025"'][:, ::2, ::2] + ymw16 = grp['"ymw16"'][:, ::2, ::2] + + logger.info( + "Loaded arrays – ne2001: %s %s, ne2025: %s %s, ymw16: %s %s", + ne2001.shape, ne2001.dtype, + ne2025.shape, ne2025.dtype, + ymw16.shape, ymw16.dtype, + ) + + except FileNotFoundError: + logger.critical("Data file not found: %s", path) + raise + except KeyError as exc: + logger.critical("Missing expected key in HDF5 file: %s", exc) + raise + except Exception: + logger.critical("Failed to load skymap data", exc_info=True) + raise + + nbytes = sum(a.nbytes for a in (ne2001, ne2025, ymw16, dist, gl, gb)) + elapsed = time.time() - t0 + logger.info("Skymap data loaded in %.2fs (%.1f MB in arrays)", elapsed, nbytes / 1024**2) + + return dist, gl, gb, ne2001, ne2025, ymw16 + + +def _build_xarray(data, dist, gl, gb): + """Wrap a numpy array as an xarray DataArray.""" + return xr.DataArray( + data, + dims=("distance_kpc", "gb", "gl"), + coords={"distance_kpc": dist, "gl": gl, "gb": gb}, + attrs={"units": "DM pc/cm3"}, + ) + + +try: + _dist, _gl, _gb, _ne2001, _ne2025, _ymw16 = load_skymap_data(DATA_PATH) + skymap_dist = _dist + skymap_data_ne = _build_xarray(_ne2001, _dist, _gl, _gb) + skymap_data_ne25 = _build_xarray(_ne2025, _dist, _gl, _gb) + skymap_data_ymw = _build_xarray(_ymw16, _dist, _gl, _gb) + del _ne2001, _ne2025, _ymw16, _dist, _gl, _gb # free raw arrays + logger.info("xarray DataArrays built successfully") +except Exception: + logger.critical("FATAL – could not initialise skymap data, exiting", exc_info=True) + sys.exit(1) # APP SETUP app = dash.Dash(__name__, external_stylesheets=[dbc.themes.SPACELAB]) @@ -206,7 +261,7 @@ def callback(n_clicks, skymap_apply_clicks, relayout_data, model, colorscale, dm sc = SkyCoord(0 * u.deg, 0 * u.deg, frame="galactic") coord_error = True - print(sc.galactic.l, sc.galactic.b, dmord, f) + logger.info("Calculating: gl=%s, gb=%s, dmord=%s, func=%s", sc.galactic.l, sc.galactic.b, dmord, f.__name__) dout = f(sc.galactic.l, sc.galactic.b, dmord, method=model, nu=nu) dout_ne = f(sc.galactic.l, sc.galactic.b, dmord, method="ne2001", nu=nu) dout_ne25 = f(sc.galactic.l, sc.galactic.b, dmord, method="ne2025", nu=nu) @@ -253,7 +308,7 @@ def callback(n_clicks, skymap_apply_clicks, relayout_data, model, colorscale, dm skymap_data = skymap_data_ymw skymap_model_label = model - print(skymap_data.shape) + logger.debug("Selected skymap model=%s, shape=%s", skymap_model_label, skymap_data.shape) # Determine DM min/max for colorscale # Only reset if distance slider changed @@ -434,7 +489,7 @@ def _to_float(val): ] about_refs = [ - html.H4("PyGEDM"), + html.H5("PyGEDM"), html.P("Price, D. C., Flynn, C., and Deller, A."), html.P([ html.A( @@ -442,7 +497,7 @@ def _to_float(val): href="https://scixplorer.org/abs/2021PASA...38...38P/abstract", ) ]), - html.H4("NE2001"), + html.H5("NE2001"), html.P("Cordes, J. M., & Lazio, T. J. W. (2002)"), html.P([ html.A( @@ -450,7 +505,7 @@ def _to_float(val): href="https://ui.adsabs.harvard.edu/abs/2002astro.ph..7156C/abstract" ) ]), - html.H4("NE2025"), + html.H5("NE2025"), html.P("Ocker, S.K. and Cordes, J.M. (2026)"), html.P([ html.A( @@ -458,7 +513,7 @@ def _to_float(val): href="https://ui.adsabs.harvard.edu/abs/2026arXiv260211838O/abstract" ) ]), - html.H4("YMW16"), + html.H5("YMW16"), html.P("Yao, J. M., Manchester, R. N., & Wang, N. (2017)"), html.P([ html.A( @@ -705,8 +760,12 @@ def _to_float(val): ]) ], width=3), dbc.Col([ - html.H5("References", className="mt-2"), - html.Div(id="about-refs"), + dbc.Card([ + dbc.CardBody([ + html.H4("References", className="card-title"), + html.Div(id="about-refs"), + ]) + ]) ], width=9), ], className="mt-3"), ]), diff --git a/app/docker-compose.yml b/app/docker-compose.yml new file mode 100644 index 0000000..1b2cdff --- /dev/null +++ b/app/docker-compose.yml @@ -0,0 +1,26 @@ +services: + pygedm: + build: + context: . + dockerfile: Dockerfile + image: pygedm-app + container_name: pygedm + ports: + - "8050:8050" + # Mount the data directory so the HKL file can be updated without + # rebuilding the image. Remove this volume if you prefer the file + # to be baked in at build time. + volumes: + - ./data:/app/data:ro + restart: unless-stopped + # Limit memory to prevent the container from consuming excessive RAM + # on the host. Adjust based on available resources. + mem_limit: 512m + environment: + - PYTHONUNBUFFERED=1 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8050')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s diff --git a/app/requirements.txt b/app/requirements.txt index 53875f5..4cc6ca7 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -12,6 +12,5 @@ gunicorn Pillow requests h5py -hickle numba git+https://github.com/telegraphic/mwprop \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0ec302d..2f0e3d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=64.0", "wheel", "pybind11>=2.2"] +requires = ["setuptools>=77.0", "wheel", "pybind11>=2.2"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +8,7 @@ version = "4.0.0" description = "Python/C++ version of NE2001, YMW16, and YT2020 electron density models" readme = "README.md" requires-python = ">=3.8" -license = {text = "MIT"} +license = "MIT" authors = [ {name = "D. C. Price", email = "dan@thetelegraphic.com"} ] @@ -16,7 +16,6 @@ keywords = ["astronomy", "radio", "pulsars", "dispersion measure"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9",