Text to SQL RFT example

Author: benjibc

text-to-sql-ep-rft/Makefile

@@ -0,0 +1,55 @@
+PYTHON ?= python
+PIP ?= pip
+
+.PHONY: install
+install:
+	$(PIP) install -r requirements.txt
+
+.PHONY: all-data
+all-data: sim-prod extract-schema synth gen-queries augment ground-truth gen-nl
+
+.PHONY: sim-prod
+sim-prod:
+	$(PYTHON) scripts/01_simulate_prod_db.py
+
+.PHONY: extract-schema
+extract-schema:
+	$(PYTHON) scripts/02_extract_schema.py
+
+.PHONY: synth
+synth:
+	$(PYTHON) scripts/03_generate_synthetic_data.py
+
+.PHONY: gen-queries
+gen-queries:
+	$(PYTHON) scripts/04_generate_queries.py
+
+.PHONY: augment
+augment:
+	$(PYTHON) scripts/05_augment_sandbox.py
+
+.PHONY: ground-truth
+ground-truth:
+	$(PYTHON) scripts/06_ground_truth.py
+
+.PHONY: gen-nl
+gen-nl:
+	$(PYTHON) scripts/07_generate_nl_questions.py
+
+.PHONY: test
+test:
+	pytest -q
+
+.PHONY: mcp-build
+mcp-build:
+	cd mcp_server && docker build -t text-to-sql-mcp:latest .
+
+.PHONY: mcp-deploy
+# Usage: make mcp-deploy PROJECT_ID=your-id REGION=us-central1
+mcp-deploy:
+	gcloud run deploy mcp-sql-rft-server \
+	  --source mcp_server \
+	  --project $(PROJECT_ID) \
+	  --region $(REGION) \
+	  --allow-unauthenticated \
+	  --port 8080

text-to-sql-ep-rft/README.md

@@ -0,0 +1,48 @@
+### Text-to-SQL RFT with Eval Protocol (End-to-End)
+
+This repository demonstrates an end-to-end Natural Language → SQL workflow using Eval Protocol and Fireworks RFT, without the old reward-kit. It includes:
+- Data generation scripts (schema-only → synthetic DB → SQL → NL → train/test)
+- A Dockerized MCP server that exposes a read-only DuckDB database over HTTP
+- An Eval Protocol evaluator that executes model-generated SQL via MCP and scores results
+- Local smoke tests and Makefile helpers
+
+#### Prerequisites
+- Python 3.11+ (recommend `uv` or `venv`)
+- `FIREWORKS_API_KEY` (Fireworks account)
+- Google Cloud SDK (for Cloud Run MCP deployment) if you want remote server
+- Optional: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY` for benchmarking additional models
+
+#### Quickstart
+1) Create a Python environment in this folder and install:
+```
+pip install -r requirements.txt
+```
+
+2) Generate data (OpenFlights → prod → synthetic → queries → ground-truth → NL):
+```
+make all-data
+```
+
+3) Build and deploy MCP server to Cloud Run:
+```
+make mcp-deploy PROJECT_ID=your-gcp-project REGION=us-central1
+```
+Copy the service URL (without trailing `/mcp/`). Set `MCP_SERVER_URL` for the evaluator.
+
+4) Test evaluator locally:
+```
+pytest -q
+```
+
+5) Launch RFT (from `evaluator/` with `.env` containing FIREWORKS_API_KEY and MCP_SERVER_URL):
+```
+cd evaluator
+eval-protocol create rft --base-model accounts/fireworks/models/qwen2p5-7b
+```
+
+6) Benchmark base vs tuned:
+```
+python scripts/benchmark_models.py
+```
+
+See `scripts/` for individual steps and `mcp_server/` for Docker deployment details.

text-to-sql-ep-rft/data/airlines.dat

@@ -0,0 +1,261 @@
+"Bonaire, Saint Eustatius and Saba","BQ",""
+"Aruba","AW","AA"
+"Antigua and Barbuda","AG","AC"
+"United Arab Emirates","AE","AE"
+"Afghanistan","AF","AF"
+"Algeria","DZ","AG"
+"Azerbaijan","AZ","AJ"
+"Albania","AL","AL"
+"Armenia","AM","AM"
+"Angola","AO","AO"
+"American Samoa","AS","AQ"
+"Argentina","AR","AR"
+"Australia","AU","AS"
+"Ashmore and Cartier Islands",\N,"AT"
+"Austria","AT","AU"
+"Anguilla","AI","AV"
+"Antarctica","AQ","AY"
+"Bahrain","BH","BA"
+"Barbados","BB","BB"
+"Botswana","BW","BC"
+"Bermuda","BM","BD"
+"Belgium","BE","BE"
+"Bahamas","BS","BF"
+"Bangladesh","BD","BG"
+"Belize","BZ","BH"
+"Bosnia and Herzegovina","BA","BK"
+"Bolivia","BO","BL"
+"Myanmar","MM","BM"
+"Benin","BJ","BN"
+"Belarus","BY","BO"
+"Solomon Islands","SB","BP"
+"Navassa Island",\N,"BQ"
+"Brazil","BR","BR"
+"India","IN","BS"
+"Bhutan","BT","BT"
+"Bulgaria","BG","BU"
+"Bouvet Island","BV","BV"
+"Brunei Darussalam","BN","BX"
+"Burundi","BI","BY"
+"Canada","CA","CA"
+"Cambodia","KH","CB"
+"Chad","TD","CD"
+"Sri Lanka","LK","CE"
+"DR Congo","CD","CF"
+"Congo Republic","CG","CG"
+"China","CN","CH"
+"Chile","CL","CI"
+"Cayman Islands","KY","CJ"
+"Cocos (Keeling) Islands","CC","CK"
+"Cameroon","CM","CM"
+"Comoros","KM","CN"
+"Colombia","CO","CO"
+"Northern Mariana Islands","MP","CQ"
+"Coral Sea Islands",\N,"CR"
+"Costa Rica","CR","CS"
+"Central African Republic","CF","CT"
+"Cuba","CU","CU"
+"Cabo Verde","CV","CV"
+"Cook Islands","CK","CW"
+"Cyprus","CY","CY"
+"Denmark","DK","DA"
+"Djibouti","DJ","DJ"
+"Dominica","DM","DO"
+"Jarvis Island",\N,"DQ"
+"Dominican Republic","DO","DR"
+"Ecuador","EC","EC"
+"Egypt","EG","EG"
+"Ireland","IE","EI"
+"Equatorial Guinea","GQ","EK"
+"Estonia","EE","EN"
+"Eritrea","ER","ER"
+"El Salvador","SV","ES"
+"Ethiopia","ET","ET"
+"Europa Island",\N,"EU"
+"Czech Republic","CZ","EZ"
+"French Guiana","GF","FG"
+"Finland","FI","FI"
+"Fiji","FJ","FJ"
+"Falkland Islands","FK","FK"
+"Micronesia, Fed. Sts.","FM","FM"
+"Faeroe Islands","FO","FO"
+"French Polynesia","PF","FP"
+"Baker Island",\N,"FQ"
+"France","FR","FR"
+"French Southern Territories","TF","FS"
+"Gambia","GM","GA"
+"Gabon","GA","GB"
+"Georgia","GE","GG"
+"Ghana","GH","GH"
+"Gibraltar","GI","GI"
+"Grenada","GD","GJ"
+"Guernsey","GG","GK"
+"Greenland","GL","GL"
+"Germany","DE","GM"
+"Glorioso Islands",\N,"GO"
+"Guadeloupe","GP","GP"
+"Guam","GU","GQ"
+"Greece","GR","GR"
+"Guatemala","GT","GT"
+"Guinea","GN","GV"
+"Guyana","GY","GY"
+"Palestine","PS","GZ"
+"Haiti","HT","HA"
+"Hong Kong","HK","HK"
+"Heard and McDonald Islands","HM","HM"
+"Honduras","HN","HO"
+"Howland Island",\N,"HQ"
+"Croatia","HR","HR"
+"Hungary","HU","HU"
+"Iceland","IS","IC"
+"Indonesia","ID","ID"
+"Isle of Man","IM","IM"
+"India","IN","IN"
+"British Indian Ocean Territory","IO","IO"
+"Clipperton Island",\N,"IP"
+"Iran","IR","IR"
+"Israel","IL","IS"
+"Italy","IT","IT"
+"Cote d'Ivoire","CI","IV"
+"Iraq","IQ","IZ"
+"Japan","JP","JA"
+"Jersey","JE","JE"
+"Jamaica","JM","JM"
+"Jan Mayen",\N,"JN"
+"Jordan","JO","JO"
+"Johnston Atoll",\N,"JQ"
+"Juan de Nova Island",\N,"JU"
+"Kenya","KE","KE"
+"Kyrgyz Republic","KG","KG"
+"North Korea","KP","KN"
+"Kingman Reef",\N,"KQ"
+"Kiribati","KI","KR"
+"South Korea","KR","KS"
+"Christmas Island","CX","KT"
+"Kuwait","KW","KU"
+"Kazakhstan","KZ","KZ"
+"Laos","LA","LA"
+"Lebanon","LB","LE"
+"Latvia","LV","LG"
+"Lithuania","LT","LH"
+"Liberia","LR","LI"
+"Slovakia","SK","LO"
+"Palmyra Atoll",\N,"LQ"
+"Lesotho","LS","LT"
+"Luxembourg","LU","LU"
+"Libya","LY","LY"
+"Madagascar","MG","MA"
+"Martinique","MQ","MB"
+"Macao","MO","MC"
+"Moldova","MD","MD"
+"Mayotte","YT","MF"
+"Mongolia","MN","MG"
+"Montserrat","MS","MH"
+"Malawi","MW","MI"
+"Montenegro","ME","MJ"
+"Macedonia","MK","MK"
+"Mali","ML","ML"
+"Monaco","MC","MN"
+"Morocco","MA","MO"
+"Mauritius","MU","MP"
+"Midway Islands",\N,"MQ"
+"Mauritania","MR","MR"
+"Malta","MT","MT"
+"Oman","OM","MU"
+"Maldives","MV","MV"
+"Mexico","MX","MX"
+"Malaysia","MY","MY"
+"Mozambique","MZ","MZ"
+"New Caledonia","NC","NC"
+"Niue","NU","NE"
+"Norfolk Island","NF","NF"
+"Niger","NE","NG"
+"Vanuatu","VU","NH"
+"Nigeria","NG","NI"
+"Netherlands","NL","NL"
+"Norway","NO","NO"
+"Nepal","NP","NP"
+"Nauru","NR","NR"
+"Suriname","SR","NS"
+"Netherlands Antilles","AN","NT"
+"Nicaragua","NI","NU"
+"New Zealand","NZ","NZ"
+"Paraguay","PY","PA"
+"Pitcairn","PN","PC"
+"Peru","PE","PE"
+"Paracel Islands",\N,"PF"
+"Spratly Islands",\N,"PG"
+"Pakistan","PK","PK"
+"Poland","PL","PL"
+"Panama","PA","PM"
+"Portugal","PT","PO"
+"Papua New Guinea","PG","PP"
+"Palau","PW","PS"
+"Guinea-Bissau","GW","PU"
+"Qatar","QA","QA"
+"Serbia","RS","RB"
+"Reunion","RE","RE"
+"Marshall Islands","MH","RM"
+"Romania","RO","RO"
+"Philippines","PH","RP"
+"Puerto Rico","PR","RQ"
+"Russia","RU","RS"
+"Rwanda","RW","RW"
+"Saudi Arabia","SA","SA"
+"St. Pierre and Miquelon","PM","SB"
+"St. Kitts and Nevis","KN","SC"
+"Seychelles","SC","SE"
+"South Africa","ZA","SF"
+"Senegal","SN","SG"
+"St. Helena","SH","SH"
+"Slovenia","SI","SI"
+"Sierra Leone","SL","SL"
+"Singapore","SG","SN"
+"Somalia","SO","SO"
+"Spain","ES","SP"
+"South Sudan","SS","SS"
+"St. Lucia","LC","ST"
+"Sudan","SD","SU"
+"Svalbard and Jan Mayen Islands","SJ","SV"
+"Sweden","SE","SW"
+"South Georgia and South Sandwich Is.","GS","SX"
+"Syria","SY","SY"
+"Switzerland","CH","SZ"
+"Trinidad and Tobago","TT","TD"
+"Tromelin Island",\N,"TE"
+"Thailand","TH","TH"
+"Tajikistan","TJ","TI"
+"Turks and Caicos Islands","TC","TK"
+"Tokelau","TK","TL"
+"Tonga","TO","TN"
+"Togo","TG","TO"
+"Sao Tome and Principe","ST","TP"
+"Tunisia","TN","TS"
+"Timor-Leste","TL","TT"
+"Turkey","TR","TU"
+"Tuvalu","TV","TV"
+"Taiwan","TW","TW"
+"Turkmenistan","TM","TX"
+"Tanzania","TZ","TZ"
+"Uganda","UG","UG"
+"United Kingdom","GB","UK"
+"Ukraine","UA","UP"
+"United States","US","US"
+"Burkina Faso","BF","UV"
+"Uruguay","UY","UY"
+"Uzbekistan","UZ","UZ"
+"St. Vincent and the Grenadines","VC","VC"
+"Venezuela","VE","VE"
+"British Virgin Islands","VG","VI"
+"Vietnam","VN","VM"
+"United States Virgin Islands","VI","VQ"
+"Namibia","NA","WA"
+"Palestine","PS","WE"
+"Wallis and Futuna Islands","WF","WF"
+"Western Sahara","EH","WI"
+"Wake Island",\N,"WQ"
+"Samoa","WS","WS"
+"Eswatini","SZ","WZ"
+"Yemen","YE","YM"
+"Zambia","ZM","ZA"
+"Zimbabwe","ZW","ZI"