Use quantized base model

Author: tedspare

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+	"biome.configurationPath": "./auth/biome.json",
+	"biome.enabled": true,
+	"biome.lsp.bin": "./auth/node_modules/.bin/biome",
+	"editor.codeActionsOnSave": {
+		"source.fixAll.biome": "explicit"
+	}
+}

auth/index.ts

@@ -3,7 +3,6 @@ import { env } from './env'
 const SGLANG_URL = 'http://localhost:8000'
 
 const server = Bun.serve({
-	port: process.env.PORT || 3000,
 	async fetch(req) {
 		const url = new URL(req.url)
 
@@ -30,23 +29,24 @@ const server = Bun.serve({
 
 			// Forward to SGLang
 			const response = await fetch(`${SGLANG_URL}${url.pathname}`, {
-				method: req.method,
+				body: req.method !== 'GET' ? await req.text() : null,
 				headers: { 'content-type': 'application/json' },
-				body: req.method !== 'GET' ? await req.text() : null
+				method: req.method
 			})
 
 			return new Response(response.body, {
-				status: response.status,
-				headers: response.headers
+				headers: response.headers,
+				status: response.status
 			})
 		} catch (error) {
 			console.error(error)
 			return Response.json(
-				{ error: 'Internal Server Error', details: JSON.stringify(error) },
+				{ details: JSON.stringify(error), error: 'Internal Server Error' },
 				{ status: 500 }
 			)
 		}
-	}
+	},
+	port: process.env.PORT || 3000
 })
 
 console.log(`Auth server running on port ${server.port}`)

skypilot.yaml

@@ -6,7 +6,7 @@ num_nodes: 1
 workdir: ./auth
 
 envs:
-  BASE_MODEL: Qwen/Qwen3-8B
+  BASE_MODEL: Qwen/Qwen3-8B-AWQ
   HF_HUB_ENABLE_HF_TRANSFER: "1"
   CUDA_HOME: "/usr/local/cuda-12"
   LD_LIBRARY_PATH: "/usr/local/cuda-12/lib64:$LD_LIBRARY_PATH"