Increase default max_attempts to 10 and add pre-processing for common… #58
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test API Endpoints | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| env: | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_API_BASE: https://coder.gaia.domains/v1 | |
| LLM_MODEL: Qwen2.5-Coder-32B-Instruct-Q5_K_M | |
| LLM_EMBED_MODEL: nomic-embed | |
| LLM_EMBED_SIZE: 768 | |
| SKIP_VECTOR_SEARCH: "true" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Install Python and dependencies | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install jq and curl | |
| run: sudo apt-get install -y jq curl | |
| - name: Install Python dependencies | |
| run: pip install -r requirements.txt | |
| - name: Run docker compose | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: ./docker-compose.yml | |
| up-flags: "-d --build" | |
| - name: Wait for services to be ready | |
| run: | | |
| echo "Waiting for services to be ready..." | |
| sleep 60 | |
| docker ps | |
| - name: Verify container readiness | |
| run: | | |
| docker ps | |
| for container in api mcp-server mcp-proxy; do | |
| if ! docker ps | grep -q "$container"; then | |
| echo "ERROR: $container is not running!" | |
| docker compose logs --tail=100 "$container" | |
| exit 1 | |
| fi | |
| done | |
| echo "All containers are running." | |
| - name: Test /compile endpoint | |
| run: | | |
| echo "Testing /compile endpoint..." | |
| RESPONSE=$(curl -s -S -f -X POST http://localhost:8000/compile \ | |
| -H "Content-Type: application/json" \ | |
| -d '{ | |
| "code": "[filename: Cargo.toml]\n[package]\nname = \"hello_world\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n\n[filename: src/main.rs]\nfn main() {\n println!(\"Hello, World!\");\n}" | |
| }' || echo "CURL_FAILED") | |
| if [ "$RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to connect to API service" | |
| docker ps | |
| docker logs $(docker ps -q --filter name=api) | |
| exit 1 | |
| fi | |
| # Check for success in response | |
| if ! echo "$RESPONSE" | jq -e '.success == true' > /dev/null; then | |
| echo "Compilation failed:" | |
| echo "$RESPONSE" | jq || echo "$RESPONSE" | |
| exit 1 | |
| fi | |
| echo "Compilation successful!" | |
| echo "$RESPONSE" | jq || echo "$RESPONSE" | |
| - name: Test /compile-and-fix endpoint | |
| run: | | |
| echo "Testing /compile-and-fix endpoint..." | |
| RESPONSE=$(curl -s -S -f -X POST http://localhost:8000/compile-and-fix \ | |
| -H "Content-Type: application/json" \ | |
| -d '{ | |
| "code": "[filename: Cargo.toml]\n[package]\nname = \"hello_world\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n\n[filename: src/main.rs]\nfn main() {\n println!(\"Hello, World!\" // Missing closing parenthesis\n}", | |
| "description": "A simple hello world program", | |
| "max_attempts": 3 | |
| }' || echo "CURL_FAILED") | |
| if [ "$RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to connect to API service" | |
| docker ps | |
| docker logs $(docker ps -q --filter name=api) | |
| exit 1 | |
| fi | |
| # Check if response is JSON (starts with {) or text format | |
| if [[ "$RESPONSE" == {* ]]; then | |
| # JSON response (likely error) | |
| echo "Got JSON response (likely error):" | |
| echo "$RESPONSE" | jq || echo "$RESPONSE" | |
| # Verify it's a valid response with attempts | |
| if echo "$RESPONSE" | jq -e '.attempts' > /dev/null; then | |
| echo "Compilation couldn't be fixed, but response format is valid" | |
| else | |
| echo "Invalid error response format" | |
| exit 1 | |
| fi | |
| else | |
| # Text response (success case) | |
| # Verify the response format has filename markers | |
| if ! echo "$RESPONSE" | grep -q "\[filename:"; then | |
| echo "Response does not contain filename markers:" | |
| echo "$RESPONSE" | |
| exit 1 | |
| fi | |
| echo "Compile and fix successful! Response contains code files in text format." | |
| fi | |
| - name: Test /generate endpoint | |
| run: | | |
| echo "Testing /generate endpoint..." | |
| # Generate the project | |
| RESPONSE=$(curl -s -S -f -X POST http://localhost:8000/generate \ | |
| -H "Content-Type: application/json" \ | |
| -d '{ | |
| "description": "A simple command-line calculator in Rust", | |
| "requirements": "Should support addition, subtraction, multiplication, and division" | |
| }' || echo "CURL_FAILED") | |
| if [ "$RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to connect to API service" | |
| docker ps | |
| docker logs $(docker ps -q --filter name=api) | |
| exit 1 | |
| fi | |
| # Extract project_id from response | |
| PROJECT_ID=$(echo "$RESPONSE" | jq -r '.project_id') | |
| echo "Project ID: $PROJECT_ID" | |
| # Poll for project completion (maximum 10 attempts, 15 seconds apart) | |
| echo "Polling for project completion..." | |
| for i in {1..10}; do | |
| echo "Checking project status (attempt $i)..." | |
| STATUS_RESPONSE=$(curl -s -S -f "http://localhost:8000/project/$PROJECT_ID" || echo "CURL_FAILED") | |
| if [ "$STATUS_RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to get project status" | |
| exit 1 | |
| fi | |
| STATUS=$(echo "$STATUS_RESPONSE" | jq -r '.status') | |
| echo "Current status: $STATUS" | |
| if [ "$STATUS" = "completed" ]; then | |
| echo "Project generation successful!" | |
| echo "$STATUS_RESPONSE" | jq | |
| break | |
| elif [ "$STATUS" = "failed" ]; then | |
| echo "Project generation failed:" | |
| echo "$STATUS_RESPONSE" | jq | |
| exit 1 | |
| fi | |
| # If still processing, wait and try again | |
| if [ $i -eq 10 ]; then | |
| echo "Project generation taking too long, exiting" | |
| exit 1 | |
| fi | |
| echo "Waiting 15 seconds before next check..." | |
| sleep 15 | |
| done | |
| # Get a file from the project to verify file access works | |
| echo "Retrieving main.rs file..." | |
| FILE_RESPONSE=$(curl -s -S -f "http://localhost:8000/project/$PROJECT_ID/files/src/main.rs" || echo "CURL_FAILED") | |
| if [ "$FILE_RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to retrieve file" | |
| exit 1 | |
| fi | |
| echo "Successfully retrieved file content:" | |
| echo "$FILE_RESPONSE" | head -10 | |
| # Test downloading the project | |
| echo "Testing project download..." | |
| DOWNLOAD_RESPONSE=$(curl -s -S -f -o "project-$PROJECT_ID.zip" "http://localhost:8000/project/$PROJECT_ID/download" || echo "CURL_FAILED") | |
| if [ "$DOWNLOAD_RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to download project" | |
| exit 1 | |
| fi | |
| # Verify zip file was created | |
| if [ ! -f "project-$PROJECT_ID.zip" ]; then | |
| echo "Project zip file not created" | |
| exit 1 | |
| fi | |
| echo "Project download successful!" | |
| ls -la "project-$PROJECT_ID.zip" | |
| - name: Test /generate-sync endpoint | |
| run: | | |
| echo "Testing /generate-sync endpoint..." | |
| RESPONSE=$(curl -s -S -f -X POST http://localhost:8000/generate-sync \ | |
| -H "Content-Type: application/json" \ | |
| -d '{ | |
| "description": "A simple hello world program in Rust", | |
| "requirements": "Print a greeting message to the console" | |
| }' || echo "CURL_FAILED") | |
| if [ "$RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to connect to API service" | |
| docker ps | |
| docker logs $(docker ps -q --filter name=api) | |
| exit 1 | |
| fi | |
| # Save response to file for later use | |
| echo "$RESPONSE" > generate_output.txt | |
| # Verify the response format has filename markers | |
| if ! echo "$RESPONSE" | grep -q "\[filename:"; then | |
| echo "Response does not contain filename markers:" | |
| echo "$RESPONSE" | head -20 | |
| exit 1 | |
| fi | |
| # Check if build succeeded | |
| if ! echo "$RESPONSE" | grep -q "# Build succeeded"; then | |
| echo "Build did not succeed:" | |
| echo "$RESPONSE" | tail -20 | |
| exit 1 | |
| fi | |
| echo "Generate-sync successful! Response contains code files in text format." | |
| echo "First 20 lines of response:" | |
| echo "$RESPONSE" | head -20 | |
| - name: "Test workflow: /generate-sync → /compile" | |
| run: | | |
| echo "Testing workflow: /generate-sync → /compile..." | |
| # Get the output from the previous step and remove the build status comment | |
| GENERATE_OUTPUT=$(cat generate_output.txt | sed '/^# Build/,$d') | |
| # Pass the cleaned generated code directly to compile | |
| COMPILE_RESPONSE=$(curl -s -S -f -X POST http://localhost:8000/compile \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"code\": $(echo "$GENERATE_OUTPUT" | jq -Rs .) | |
| }" || echo "CURL_FAILED") | |
| if [ "$COMPILE_RESPONSE" = "CURL_FAILED" ]; then | |
| echo "Failed to connect to API service" | |
| docker ps | |
| docker logs $(docker ps -q --filter name=api) | |
| exit 1 | |
| fi | |
| # Check for success in response | |
| if ! echo "$COMPILE_RESPONSE" | jq -e '.success == true' > /dev/null; then | |
| echo "Compilation failed:" | |
| echo "$COMPILE_RESPONSE" | jq || echo "$COMPILE_RESPONSE" | |
| exit 1 | |
| fi | |
| echo "Workflow test successful! Generated code compiles correctly." | |
| echo "$COMPILE_RESPONSE" | jq || echo "$COMPILE_RESPONSE" | |
| - name: Check Docker logs on failure | |
| if: failure() | |
| run: | | |
| echo "Checking Docker logs for troubleshooting..." | |
| docker compose logs || echo "Failed to get logs" |