From b437e597477a8074c30c95053fa4dc178ce3d5e5 Mon Sep 17 00:00:00 2001 From: Todd White Date: Sun, 28 Jun 2026 17:54:39 -0400 Subject: [PATCH 1/3] CAGLTexture: heap-allocate the pixel buffer in -_writeToPNG: -_writeToPNG: read the texture into a stack VLA, char pixels[[self width]*[self height]*4]. For a 1024x1024 texture that is 4 MB on the stack -- a stack overflow. Allocate it on the heap and free it. --- Source/GLHelpers/CAGLTexture.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/GLHelpers/CAGLTexture.m b/Source/GLHelpers/CAGLTexture.m index 8b91e1f..10e2e06 100644 --- a/Source/GLHelpers/CAGLTexture.m +++ b/Source/GLHelpers/CAGLTexture.m @@ -254,7 +254,9 @@ - (GLenum) textureTarget - (void) _writeToPNG:(NSString*)path { - char pixels[[self width]*[self height]*4]; + char *pixels = malloc([self width]*[self height]*4); + if (pixels == NULL) + return; [self bind]; glGetTexImage([self textureTarget], 0, GL_RGBA, GL_UNSIGNED_BYTE, pixels); @@ -271,5 +273,6 @@ - (void) _writeToPNG:(NSString*)path CGImageRelease(image); [data writeToFile:path atomically:YES]; + free(pixels); } @end From d46b153be7f130366e4c0c04514a609c17e82041 Mon Sep 17 00:00:00 2001 From: Todd White Date: Mon, 29 Jun 2026 08:52:23 -0400 Subject: [PATCH 2/3] CAGLTexture: free the pixel buffer right after the image is released Move free(pixels) to immediately after CGImageRelease(image), before the file is written, so a future early return added around the file write cannot leak the buffer (per review feedback). The buffer cannot be freed any earlier: CGBitmapContextCreate uses it in place as the bitmap backing store and CGBitmapContextCreateImage returns an image that references it through a data provider (no snapshot), so it must stay live until CGImageRelease. --- Source/GLHelpers/CAGLTexture.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/GLHelpers/CAGLTexture.m b/Source/GLHelpers/CAGLTexture.m index 10e2e06..9af62cb 100644 --- a/Source/GLHelpers/CAGLTexture.m +++ b/Source/GLHelpers/CAGLTexture.m @@ -271,8 +271,8 @@ - (void) _writeToPNG:(NSString*)path CGImageDestinationAddImage(destination, image, NULL); CGImageDestinationFinalize(destination); CGImageRelease(image); + free(pixels); [data writeToFile:path atomically:YES]; - free(pixels); } @end From 40ab710671632bb6287774f48e8a8967d5ecfca5 Mon Sep 17 00:00:00 2001 From: Todd White Date: Mon, 29 Jun 2026 15:46:49 -0400 Subject: [PATCH 3/3] Tests: add a headless regression test for CAGLTexture -_writeToPNG: Add Tests/headless, a display-free test that creates a surfaceless EGL desktop-GL context (Mesa software rendering) and exercises CAGLTexture -_writeToPNG: directly, obtaining the class with objc_getClass: - a texture round-trips to a valid PNG of the expected dimensions; - writing a large texture does not overflow the stack -- the previous implementation read the image into a stack VLA (char pixels[w*h*4]). The large-texture case runs on a thread with a deliberately small stack so the regression is deterministic regardless of the caller's stack limit. The test only builds when EGL is available (pkg-config egl) and skips at run time if no GL context can be created, so it is inert where headless GL is unavailable. Run it with "make check" in Tests/headless. --- Tests/headless/GNUmakefile | 36 +++++++ Tests/headless/headless_texture.m | 164 ++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 Tests/headless/GNUmakefile create mode 100644 Tests/headless/headless_texture.m diff --git a/Tests/headless/GNUmakefile b/Tests/headless/GNUmakefile new file mode 100644 index 0000000..ef38256 --- /dev/null +++ b/Tests/headless/GNUmakefile @@ -0,0 +1,36 @@ +# Headless regression tests for QuartzCore. +# +# These require a desktop-GL context created without a display, via a +# surfaceless EGL context (Mesa/llvmpipe). They are therefore only built when +# EGL is available; on platforms without it the target is skipped. +# +# Run with: make check + +include $(GNUSTEP_MAKEFILES)/common.make + +HAVE_EGL := $(shell pkg-config --exists egl 2>/dev/null && echo yes) + +QC_LIBDIR := $(CURDIR)/../../Source/QuartzCore.framework/Versions/0 + +ifeq ($(HAVE_EGL),yes) + +TOOL_NAME = headless_texture +headless_texture_OBJC_FILES = headless_texture.m + +ADDITIONAL_OBJCFLAGS += $(shell pkg-config --cflags egl) +ADDITIONAL_LIB_DIRS += -L$(QC_LIBDIR) +ADDITIONAL_TOOL_LIBS += -lQuartzCore $(shell pkg-config --libs egl) -lGL -lpthread + +endif + +include $(GNUSTEP_MAKEFILES)/tool.make + +ifeq ($(HAVE_EGL),yes) +check:: all + @echo "Running headless QuartzCore tests..." + @LD_LIBRARY_PATH="$(QC_LIBDIR):$(LD_LIBRARY_PATH)" \ + $(GNUSTEP_OBJ_DIR)/headless_texture +else +check:: + @echo "EGL not found (pkg-config egl); skipping headless QuartzCore tests." +endif diff --git a/Tests/headless/headless_texture.m b/Tests/headless/headless_texture.m new file mode 100644 index 0000000..cdf3451 --- /dev/null +++ b/Tests/headless/headless_texture.m @@ -0,0 +1,164 @@ +/* Headless regression tests for CAGLTexture -_writeToPNG:. + + These run entirely without a display: a surfaceless EGL context backed by + Mesa's software rasteriser (llvmpipe) provides a desktop-GL context, which + is all CAGLTexture needs (it only uploads a texture and reads it back with + glGetTexImage -- there is no on-screen rendering). The class is obtained + with objc_getClass so no private header is required. + + Build is gated on EGL being available (see GNUmakefile); at run time the + test skips cleanly if no GL context can be created. + + Test 1 exercises the full readback + encode path and checks a valid PNG of + the right dimensions is produced. + + Test 2 is the regression for the stack overflow: the previous + implementation read the whole texture into a stack VLA + (char pixels[width*height*4]); for a large texture that overflowed the + stack. Running it on a thread with a deliberately small stack makes the + overflow deterministic, so a regression aborts the process here instead of + depending on the caller's stack limit. */ + +#import +#import +#include +#include +#include +#include + +#ifndef EGL_PLATFORM_SURFACELESS_MESA +#define EGL_PLATFORM_SURFACELESS_MESA 0x31DD +#endif +#include +#include +#include + +@interface CAGLTexture : NSObject ++ (instancetype) texture; +- (void) loadRGBATexImage:(void*)data width:(unsigned int)w height:(unsigned int)h; +- (void) _writeToPNG:(NSString*)path; +@end + +static int failures = 0; +static void check(int cond, const char *msg) +{ + printf("%s: %s\n", cond ? "PASS" : "FAIL", msg); + if (!cond) failures++; +} + +static int makeHeadlessGL(void) +{ + typedef EGLDisplay (*getPlatDisp)(EGLenum, void*, const EGLint*); + getPlatDisp gpd = (getPlatDisp)eglGetProcAddress("eglGetPlatformDisplayEXT"); + EGLDisplay dpy = gpd ? gpd(EGL_PLATFORM_SURFACELESS_MESA, EGL_DEFAULT_DISPLAY, NULL) + : eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (dpy == EGL_NO_DISPLAY || !eglInitialize(dpy, NULL, NULL)) + return 0; + if (!eglBindAPI(EGL_OPENGL_API)) /* desktop GL: glGetTexImage is not in GLES */ + return 0; + EGLint cfgAttrs[] = { EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, EGL_NONE }; + EGLConfig cfg; EGLint n = 0; + if (!eglChooseConfig(dpy, cfgAttrs, &cfg, 1, &n) || n < 1) + return 0; + EGLContext ctx = eglCreateContext(dpy, cfg, EGL_NO_CONTEXT, NULL); + if (ctx == EGL_NO_CONTEXT) + return 0; + if (!eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, ctx)) + return 0; + return 1; +} + +static unsigned be32(const unsigned char *p) +{ + return ((unsigned)p[0] << 24) | ((unsigned)p[1] << 16) + | ((unsigned)p[2] << 8) | (unsigned)p[3]; +} + +static NSString *tmpPath(NSString *name) +{ + return [NSTemporaryDirectory() stringByAppendingPathComponent: name]; +} + +/* Run on a small-stack thread so the old stack-VLA reliably overflows. */ +static void *largeTextureThread(void *arg) +{ + unsigned dim = *(unsigned *)arg; + @autoreleasepool + { + Class cls = objc_getClass("CAGLTexture"); + unsigned char *up = malloc((size_t)dim * dim * 4); + if (up == NULL) + return (void *)1; /* out of memory: treat as non-fatal */ + memset(up, 0x40, (size_t)dim * dim * 4); + id tex = [cls texture]; + [tex loadRGBATexImage: up width: dim height: dim]; + [tex _writeToPNG: tmpPath(@"qc_headless_large.png")]; + free(up); + } + return (void *)1; +} + +int main(void) +{ + @autoreleasepool + { + if (!makeHeadlessGL()) + { + printf("SKIP: no headless GL context available\n"); + return 0; + } + printf("GL: %s / %s\n", glGetString(GL_VERSION), glGetString(GL_RENDERER)); + + Class cls = objc_getClass("CAGLTexture"); + check(cls != Nil, "CAGLTexture class is available"); + if (cls == Nil) + return 1; + + /* Test 1: a texture round-trips to a valid PNG of the right size. */ + { + const unsigned W = 64, H = 48; + unsigned char *up = malloc((size_t)W * H * 4); + unsigned i; + for (i = 0; i < W * H * 4; i++) + up[i] = (unsigned char)(i * 131 + 7); + + id tex = [cls texture]; + [tex loadRGBATexImage: up width: W height: H]; + NSString *path = tmpPath(@"qc_headless_64x48.png"); + [tex _writeToPNG: path]; + + NSData *png = [NSData dataWithContentsOfFile: path]; + const unsigned char *b = (const unsigned char *)[png bytes]; + int sig = (png && [png length] > 24 + && b[0] == 0x89 && b[1] == 'P' && b[2] == 'N' && b[3] == 'G'); + check(sig, "writeToPNG produced a valid PNG"); + if (sig) + check(be32(b + 16) == W && be32(b + 20) == H, + "PNG dimensions match the texture"); + free(up); + } + + /* Test 2: regression -- a large texture must not be read into a stack + VLA. Run on a 2 MB-stack thread; the old 4 MB VLA overflows it. */ + { + unsigned dim = 1024; /* 1024*1024*4 = 4 MB readback buffer */ + pthread_attr_t attr; + pthread_t th; + void *res = NULL; + if (pthread_attr_init(&attr) == 0 + && pthread_attr_setstacksize(&attr, 2 * 1024 * 1024) == 0 + && pthread_create(&th, &attr, largeTextureThread, &dim) == 0) + { + pthread_join(th, &res); + check(res != NULL, + "writeToPNG of a large texture did not overflow a 2 MB stack"); + } + pthread_attr_destroy(&attr); + } + + printf("\n%s (%d failure%s)\n", + failures ? "FAILED" : "All OK", failures, failures == 1 ? "" : "s"); + return failures ? 1 : 0; + } +}