compress: transparent gzip-compression (#545)

.gitignore

@@ -31,6 +31,9 @@
 /common/lib/stb_image.h
 /common/cc-runtime.s2.c
 /cc-runtime
+/common/compress/pdgzip.c
+/common/compress/pdgzip.h
+/pdgzip
 /libfdt
 /edk2-ovmf
 /bochsout.txt

3RDPARTY.md

@@ -58,6 +58,9 @@ used for wallpaper image loading.
 - [libfdt](https://github.com/osdev0/libfdt) (BSD-2-Clause) is used for
 manipulating Flat Device Trees.
 
+- [pdgzip](https://github.com/iczelia/pdgzip) (0BSD) is used to provide the
+transparent gzip decompression layer for loaded files.
+
 Note that some of these projects, or parts of them, are provided under
 dual-licensing, in which case, in the above list, the only license mentioned is
 the one chosen by the Limine developers. Refer to each individual project's

GNUmakefile.in

@@ -209,8 +209,8 @@ $(call MKESCAPE,$(BINDIR))/limine-uefi-cd.bin: $(if $(BUILD_UEFI_IA32),$(call MK
 ifneq ($(BUILD_UEFI_CD),no)
 	$(MKDIR_P) '$(call SHESCAPE,$(BINDIR))'
 	rm -f '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin'
-	dd if=/dev/zero of='$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' bs=512 count=5760 2>/dev/null
-	mformat -i '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' -f 2880 -N 12345678 ::
+	dd if=/dev/zero of='$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' bs=512 count=32768 2>/dev/null
+	mformat -i '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' -h 64 -t 32 -s 16 -N 12345678 ::
 	LIMINE_UEFI_CD_TMP="$$(mktemp -d)"; \
 		mkdir -p "$$LIMINE_UEFI_CD_TMP"/EFI/BOOT; \
 		cp '$(call SHESCAPE,$(BUILDDIR))/common-uefi-aarch64/BOOTAA64.EFI' "$$LIMINE_UEFI_CD_TMP"/EFI/BOOT/ 2>/dev/null; \

bootstrap

@@ -81,6 +81,13 @@ if ! test -f version; then
         dae79833b57a01b9fd3e359ee31def69f5ae899b
     cp cc-runtime/src/cc-runtime.c common/cc-runtime.s2.c
 
+    clone_repo_commit \
+        https://github.com/iczelia/pdgzip.git \
+        pdgzip \
+        16c41d9af067c4185c136622c58ad4188609a3d1
+    cp pdgzip/pdgzip.c common/compress/pdgzip.c
+    cp pdgzip/pdgzip.h common/compress/pdgzip.h
+
     clone_repo_commit \
         https://github.com/Limine-Bootloader/limine-protocol.git \
         limine-protocol \

common/compress/gzip.c

@@ -0,0 +1,146 @@
+/* Limine glue around pdgzip: transparent gzip decompression layer over a
+ * file_handle.  The underlying decoder lives in common/compress/pdgzip.c
+ * (imported by ./bootstrap from the upstream iczelia/pdgzip repo); this
+ * file only wires pdgzip's streaming read-callback API into Limine's
+ * file_handle abstraction and adds support for random-access reads via
+ * rewind-and-skip.
+ *
+ * Copyright (C) 2019-2026 Mintsuki and contributors.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <lib/libc.h>
+#include <lib/misc.h>
+#include <lib/print.h>
+#include <mm/pmm.h>
+#include <compress/gzip.h>
+#include <compress/pdgzip.h>
+
+struct gzip_handle {
+  struct file_handle * source;     /*  compressed file (owned)  */
+  pdgzip_t           * gz;         /*  decoder backed by `scratch`  */
+  void               * scratch;    /*  pdgzip scratch buffer  */
+  size_t               scratch_sz;
+  uint64_t             src_pos;    /*  next byte to pull from `source`  */
+  uint64_t             dec_pos;    /*  current decompressed stream offset  */
+};
+
+/*  pdgzip read callback: pull up to `len` bytes from the compressed source
+    starting at gh->src_pos.  A short read (including zero) signals EOF to
+    the decoder, which is correct at the end of the file.  */
+static size_t gz_source_read(void * user, void * buf, size_t len) {
+  struct gzip_handle * gh = user;
+  uint64_t avail = gh->source->size - gh->src_pos;
+  if ((uint64_t)len > avail) len = (size_t)avail;
+  if (len == 0) return 0;
+  fread(gh->source, buf, gh->src_pos, len);
+  gh->src_pos += len;
+  return len;
+}
+
+/*  (Re)initialize the decoder for a fresh pass over the compressed stream.
+    pdgzip_init zeroes its own scratch, so we only need to reset our own
+    bookkeeping.  */
+static void gz_reset(struct gzip_handle * gh) {
+  pdgzip_cfg_t cfg = { .read = gz_source_read, .user = gh, .concat = 0 };
+  gh->src_pos = 0;
+  gh->dec_pos = 0;
+  gh->gz = pdgzip_init(gh->scratch, &cfg);
+}
+
+static uint64_t gzip_read(struct file_handle * file, void * buf, uint64_t loc, uint64_t count) {
+  struct gzip_handle * gh = file->fd;
+  /*  Rewind on backward seeks.  */
+  if (loc < gh->dec_pos) gz_reset(gh);
+  /*  Skip forward to reach the requested offset.  EOS during seek means
+      the requested location is past end-of-stream - return 0 bytes.  */
+  while (gh->dec_pos < loc) {
+    uint8_t discard[4096];
+    uint64_t gap = loc - gh->dec_pos;
+    size_t chunk = gap > sizeof(discard) ? sizeof(discard) : (size_t)gap;
+    int64_t n = pdgzip_read(gh->gz, discard, chunk);
+    if (n < 0) panic(false, "gzip: decompression error during seek");
+    if (n == 0) return 0;
+    gh->dec_pos += (uint64_t)n;
+  }
+  /*  Decompress the requested data.  */
+  uint8_t * dst = buf;
+  uint64_t remaining = count;
+  while (remaining > 0) {
+    size_t chunk = remaining > 65536 ? 65536 : (size_t)remaining;
+    int64_t n = pdgzip_read(gh->gz, dst, chunk);
+    if (n < 0) panic(false, "gzip: decompression error");
+    if (n == 0) break;
+    dst += n;
+    remaining -= (uint64_t)n;
+    gh->dec_pos += (uint64_t)n;
+  }
+  return count - remaining;
+}
+
+static void gzip_close(struct file_handle * file) {
+  struct gzip_handle * gh = file->fd;
+  fclose(gh->source);
+  pmm_free(gh->scratch, gh->scratch_sz);
+  pmm_free(gh, sizeof(struct gzip_handle));
+}
+
+bool gzip_check(struct file_handle * fd) {
+  if (fd->size < 18) return false;
+  uint8_t magic[2];  fread(fd, magic, 0, 2);
+  return magic[0] == 0x1F && magic[1] == 0x8B;
+}
+
+struct file_handle * gzip_open(struct file_handle * compressed) {
+  /*  The decompressed size is not known up front.  The 4-byte ISIZE trailer
+      is unreliable (modulo 2^32, spec defect) and callers must instead
+      drain until gzip_read returns 0 bytes (EOS).  Advertise an unknown
+      size via UINT64_MAX.  */
+  struct gzip_handle * gh = ext_mem_alloc(sizeof(struct gzip_handle));
+  gh->source     = compressed;
+  gh->scratch_sz = pdgzip_state_size();
+  gh->scratch    = ext_mem_alloc(gh->scratch_sz);
+  gz_reset(gh);
+  /*  Depends on ext_mem_alloc returning zeroed memory.  */
+  struct file_handle * ret = ext_mem_alloc(sizeof(struct file_handle));
+  ret->fd = gh;
+  ret->read = (void *) gzip_read;
+  ret->close = (void *) gzip_close;
+  ret->size = UINT64_MAX;
+  ret->vol = compressed->vol;
+  if (compressed->path != NULL && compressed->path_len > 0) {
+    ret->path = ext_mem_alloc(compressed->path_len);
+    memcpy(ret->path, compressed->path, compressed->path_len);
+    ret->path_len = compressed->path_len;
+  }
+#if defined (UEFI)
+  ret->efi_part_handle = compressed->efi_part_handle;
+#endif
+  ret->pxe = compressed->pxe;
+  ret->pxe_ip = compressed->pxe_ip;
+  ret->pxe_port = compressed->pxe_port;
+  return ret;
+}

common/compress/gzip.h

@@ -0,0 +1,53 @@
+/* embeddable gzip decoder: Copyright (C) 2026 Kamila Szewczyk <k@iczelia.net>
+ * limine: Copyright (C) 2019-2026 Mintsuki and contributors.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef COMPRESS__GZIP_H__
+#define COMPRESS__GZIP_H__
+
+#include <fs/file.h>
+
+/* Check if a file handle points to gzip-compressed data (0x1F 0x8B magic). */
+bool gzip_check(struct file_handle * fd);
+
+/* Wrap a gzip-compressed file handle in a decompressing layer.
+ *
+ * Returns a new file_handle whose read callback transparently
+ * decompresses the data.  The returned handle takes ownership of
+ * `compressed` and will close it when itself is closed.
+ *
+ * WARNING: Due to a Gzip format deficiency, ->size of the resulting
+ * file_handle is only an approximation (i.e., it is not correct for
+ * files larger than 4 GiB and doesn't necessarily have to reflect
+ * the genuine decompressed size at all in adversarial circumstances).
+ * 
+ * The real decompressed size can only be authoritatively obtained by
+ * fully decompressing the file.
+ *
+ * Supports very fast sequential reads and random-access reads (with
+ * an implicit rewind + skip penalty inherent to the gzip format).
+ */
+struct file_handle * gzip_open(struct file_handle * compressed);
+
+#endif

common/crypt/blake2b.c

@@ -6,6 +6,8 @@
 #include <stddef.h>
 #include <crypt/blake2b.h>
 #include <lib/libc.h>
+#include <lib/misc.h>
+#include <mm/pmm.h>
 
 #define BLAKE2B_BLOCK_BYTES 128
 #define BLAKE2B_KEY_BYTES 64
@@ -218,3 +220,73 @@ void blake2b(void *out, const void *in, size_t in_len) {
     blake2b_update(&state, in, in_len);
     blake2b_final(&state, out);
 }
+
+/*  Streaming filter: wraps a source file_handle and hashes bytes as
+    they are read sequentially. The hash is finalized and compared via
+    blake2b_check_hash(). Non-sequential reads panic -- the filter is
+    meant to sit underneath the gzip bitreader or uri_open's drain loop,
+    both of which advance monotonically.  */
+struct blake2b_handle {
+    struct file_handle  *source;
+    struct blake2b_state state;
+    uint64_t             pos;
+    bool                 finalized;
+    uint8_t              digest[BLAKE2B_OUT_BYTES];
+};
+
+static uint64_t blake2b_read(struct file_handle *fh, void *buf, uint64_t loc, uint64_t count) {
+    struct blake2b_handle *h = fh->fd;
+    if (loc != h->pos) {
+        panic(false, "blake2b filter: non-sequential read (pos=%x, loc=%x)",
+              (uint64_t)h->pos, loc);
+    }
+    uint64_t got = fread(h->source, buf, loc, count);
+    blake2b_update(&h->state, buf, got);
+    h->pos += got;
+    return got;
+}
+
+static void blake2b_close(struct file_handle *fh) {
+    struct blake2b_handle *h = fh->fd;
+    fclose(h->source);
+    pmm_free(h, sizeof(struct blake2b_handle));
+}
+
+struct file_handle *blake2b_open(struct file_handle *source) {
+    struct blake2b_handle *h = ext_mem_alloc(sizeof(struct blake2b_handle));
+    blake2b_init(&h->state);
+    h->source = source;
+    h->pos = 0;
+    h->finalized = false;
+
+    struct file_handle *ret = ext_mem_alloc(sizeof(struct file_handle));
+    ret->fd = h;
+    ret->read = (void *)blake2b_read;
+    ret->close = (void *)blake2b_close;
+    ret->size = source->size;
+    ret->vol = source->vol;
+    if (source->path != NULL && source->path_len > 0) {
+        ret->path = ext_mem_alloc(source->path_len);
+        memcpy(ret->path, source->path, source->path_len);
+        ret->path_len = source->path_len;
+    }
+#if defined (UEFI)
+    ret->efi_part_handle = source->efi_part_handle;
+#endif
+    ret->pxe = source->pxe;
+    ret->pxe_ip = source->pxe_ip;
+    ret->pxe_port = source->pxe_port;
+    return ret;
+}
+
+bool blake2b_check_hash(struct file_handle *fh, void *reference_hash) {
+    if (fh->read != (void *)blake2b_read) {
+        panic(false, "blake2b_check_hash: not a blake2b filter handle");
+    }
+    struct blake2b_handle *h = fh->fd;
+    if (!h->finalized) {
+        blake2b_final(&h->state, h->digest);
+        h->finalized = true;
+    }
+    return memcmp(h->digest, reference_hash, BLAKE2B_OUT_BYTES) == 0;
+}

common/crypt/blake2b.h

@@ -7,4 +7,8 @@
 
 void blake2b(void *out, const void *in, size_t in_len);
 
+struct file_handle;
+struct file_handle * blake2b_open(struct file_handle * source);
+bool blake2b_check_hash(struct file_handle *fd, void* reference_hash);
+
 #endif

common/fs/fat32.s2.c

@@ -605,7 +605,7 @@ char *fat32_get_label(struct volume *part) {
     return context.label;
 }
 
-static void fat32_read(struct file_handle *handle, void *buf, uint64_t loc, uint64_t count);
+static uint64_t fat32_read(struct file_handle *handle, void *buf, uint64_t loc, uint64_t count);
 static void fat32_close(struct file_handle *file);
 
 struct file_handle *fat32_open(struct volume *part, const char *path) {
@@ -622,7 +622,7 @@ struct file_handle *fat32_open(struct volume *part, const char *path) {
     unsigned int current_index = 0;
     char current_part[FAT32_LFN_MAX_FILENAME_LENGTH];
 
-    // skip trailing slashes
+    // skip leading slashes
     while (path[current_index] == '/') {
         current_index++;
     }
@@ -719,11 +719,12 @@ struct file_handle *fat32_open(struct volume *part, const char *path) {
     }
 }
 
-static void fat32_read(struct file_handle *file, void *buf, uint64_t loc, uint64_t count) {
+static uint64_t fat32_read(struct file_handle *file, void *buf, uint64_t loc, uint64_t count) {
     struct fat32_file_handle *f = file->fd;
     if (!read_cluster_chain(&f->context, f->cluster_chain, f->chain_len, buf, loc, count)) {
         panic(false, "fat32: cluster chain read failed (corrupted filesystem?)");
     }
+    return count;
 }
 
 static void fat32_close(struct file_handle *file) {