Skip to content

Commit c47887b

Browse files
author
Krzysztof Dziedzic
committed
test: set up itk nightly runs
1 parent c8f3df3 commit c47887b

4 files changed

Lines changed: 262 additions & 78 deletions

File tree

‎.github/workflows/nightly.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Nightly ITK
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *' # 2:00 AM UTC daily
6+
workflow_dispatch: # Allow manual execution
7+
push:
8+
branches:
9+
- 'dziedzick/itk-nighly'
10+
- 'dziedzick/itk-nightyly'
11+
12+
permissions:
13+
contents: write # Crucial permission to publish release metrics assets
14+
15+
jobs:
16+
nightly:
17+
name: Nightly ITK Run
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Checkout code
22+
uses: actions/checkout@v6
23+
24+
- name: Install uv
25+
uses: astral-sh/setup-uv@v7
26+
27+
- name: Run Nightly ITK Tests
28+
run: bash run_itk.sh
29+
working-directory: itk
30+
env:
31+
A2A_SAMPLES_REVISION: itk-v.021-alpha
32+
ITK_NIGHTLY_RUN: "True"
33+
34+
- name: Upload Results to Rolling Release
35+
uses: softprops/action-gh-release@v2
36+
with:
37+
tag_name: "nightly-metrics"
38+
prerelease: true
39+
files: |
40+
itk/itk_python.json
41+
env:
42+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎itk/process_results.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/usr/bin/env python3
2+
import os
3+
import sys
4+
import json
5+
import datetime
6+
import logging
7+
import urllib.request
8+
import urllib.error
9+
10+
# --- CONSTANTS ---
11+
RESULTS_FILE = 'raw_results.json'
12+
HISTORY_OUTPUT_FILE = 'itk_python.json'
13+
HISTORY_URL = 'https://github.com/a2aproject/a2a-python/releases/download/nightly-metrics/itk_python.json'
14+
SCENARIOS_FILE = 'scenarios.json'
15+
DEFAULT_HISTORY_LIMIT = 50
16+
17+
# Configure logging to match standard ITK formatting
18+
logging.basicConfig(
19+
level=logging.INFO,
20+
)
21+
logger = logging.getLogger(__name__)
22+
23+
24+
def main():
25+
# 1. Load raw compatibility results
26+
if not os.path.exists(RESULTS_FILE):
27+
logger.error('Results file %s not found.', RESULTS_FILE)
28+
sys.exit(1)
29+
30+
try:
31+
with open(RESULTS_FILE, 'r') as f:
32+
data = json.load(f)
33+
except Exception as e:
34+
logger.error('Error loading results JSON: %s', e)
35+
sys.exit(1)
36+
37+
all_passed = data.get('all_passed', False)
38+
results = data.get('results', {})
39+
40+
# --- NIGHTLY PIPELINE HISTORY PROCESSING ---
41+
logger.info(
42+
'Nightly run detected. Fetching existing history from GitHub releases...'
43+
)
44+
45+
# 2. Fetch existing history
46+
try:
47+
req = urllib.request.Request(
48+
HISTORY_URL, headers={'User-Agent': 'Mozilla/5.0'}
49+
)
50+
with urllib.request.urlopen(req, timeout=15) as response:
51+
if response.status == 200:
52+
history = json.loads(response.read().decode('utf-8'))
53+
logger.info(
54+
'Successfully retrieved history. Current entries: %d',
55+
len(history),
56+
)
57+
else:
58+
logger.warning(
59+
'No existing history found (HTTP %d). Initializing fresh history.',
60+
response.status,
61+
)
62+
history = []
63+
except urllib.error.HTTPError as e:
64+
if e.code == 444 or e.code == 404:
65+
logger.warning(
66+
'No existing history found (HTTP %d). Initializing fresh history.',
67+
e.code,
68+
)
69+
else:
70+
logger.warning(
71+
'HTTP error downloading existing history: %d. Initializing fresh history.',
72+
e.code,
73+
)
74+
history = []
75+
except Exception as e:
76+
logger.warning(
77+
'Failed to download existing history: %s. Initializing fresh history.',
78+
e,
79+
)
80+
history = []
81+
82+
# 3. Load full scenario definitions to compile comprehensive data
83+
try:
84+
with open(SCENARIOS_FILE, 'r') as f:
85+
scenarios_data = json.load(f)
86+
scenarios_list = scenarios_data.get('tests', [])
87+
except Exception as e:
88+
logger.warning('Failed to load scenarios.json definitions: %s', e)
89+
scenarios_list = []
90+
91+
# Merge full definitions with pass/fail outcome
92+
compiled_scenarios = []
93+
for scenario in scenarios_list:
94+
name = scenario.get('name')
95+
passed = results.get(name, False)
96+
combined = dict(scenario)
97+
combined['passed'] = passed
98+
compiled_scenarios.append(combined)
99+
100+
# 4. Compile new run data
101+
new_run = {
102+
'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(),
103+
'commit_sha': os.environ.get('GITHUB_SHA', 'local-dev'),
104+
'github_run_id': os.environ.get('GITHUB_RUN_ID', '0'),
105+
'all_passed': all_passed,
106+
'scenarios': compiled_scenarios,
107+
}
108+
109+
# 5. Merge and Prune
110+
history.append(new_run)
111+
112+
# Keep last N entries (default to 50, configurable)
113+
history_limit = int(
114+
os.environ.get('ITK_HISTORY_LIMIT', str(DEFAULT_HISTORY_LIMIT))
115+
)
116+
if len(history) > history_limit:
117+
history = history[-history_limit:]
118+
logger.info('Pruned history to last %d entries.', history_limit)
119+
120+
# 6. Save to disk (for upload step)
121+
try:
122+
with open(HISTORY_OUTPUT_FILE, 'w') as f:
123+
json.dump(history, f, indent=2)
124+
logger.info(
125+
'Successfully compiled and wrote nightly history to: %s',
126+
HISTORY_OUTPUT_FILE,
127+
)
128+
except Exception as e:
129+
logger.error('Error writing history file: %s', e)
130+
sys.exit(1)
131+
132+
133+
if __name__ == '__main__':
134+
main()

‎itk/run_itk.sh‎

Lines changed: 14 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -112,83 +112,18 @@ fi
112112
echo "ITK Service is up! Sending compatibility test request..."
113113
RESPONSE=$(curl -s -X POST http://127.0.0.1:8000/run \
114114
-H "Content-Type: application/json" \
115-
-d '{
116-
"tests": [
117-
{
118-
"name": "Star Topology (Full) - JSONRPC & GRPC",
119-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
120-
"traversal": "euler",
121-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
122-
"protocols": ["jsonrpc", "grpc"],
123-
"behavior": "send_message"
124-
},
125-
{
126-
"name": "Star Topology (No Go v03) - HTTP_JSON",
127-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
128-
"traversal": "euler",
129-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
130-
"protocols": ["http_json"],
131-
"behavior": "send_message"
132-
},
133-
{
134-
"name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
135-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
136-
"traversal": "euler",
137-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
138-
"protocols": ["jsonrpc", "grpc"],
139-
"streaming": true,
140-
"behavior": "send_message"
141-
},
142-
{
143-
"name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
144-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
145-
"traversal": "euler",
146-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
147-
"protocols": ["http_json"],
148-
"streaming": true,
149-
"behavior": "send_message"
150-
},
151-
{
152-
"name": "Push Notification Test - JSONRPC & GRPC",
153-
"sdks": ["current", "python_v10", "python_v03", "go_v03"],
154-
"traversal": "euler",
155-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
156-
"protocols": ["jsonrpc", "grpc"],
157-
"behavior": "push_notification"
158-
},
159-
{
160-
"name": "Push Notification Test - HTTP_JSON",
161-
"sdks": ["current", "python_v10", "python_v03"],
162-
"traversal": "euler",
163-
"edges": ["0->1", "0->2", "1->0", "2->0"],
164-
"protocols": ["http_json"],
165-
"behavior": "push_notification"
166-
},
167-
{
168-
"name": "Resubscribe Test - JSONRPC",
169-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
170-
"traversal": "euler",
171-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
172-
"protocols": ["jsonrpc"],
173-
"streaming": true,
174-
"behavior": "resubscribe"
175-
},
176-
{
177-
"name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
178-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
179-
"traversal": "euler",
180-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
181-
"protocols": ["grpc", "http_json"],
182-
"streaming": true,
183-
"behavior": "resubscribe"
184-
}
185-
]
186-
}')
187-
188-
echo "--------------------------------------------------------"
189-
echo "ITK TEST RESULTS:"
190-
echo "--------------------------------------------------------"
191-
echo "$RESPONSE" | python3 -c "
115+
-d @scenarios.json)
116+
117+
if [ "${ITK_NIGHTLY_RUN^^}" = "TRUE" ]; then
118+
echo "Nightly run detected. Saving raw results and running process_results.py..."
119+
echo "$RESPONSE" > raw_results.json
120+
python3 process_results.py
121+
RESULT=$?
122+
else
123+
echo "--------------------------------------------------------"
124+
echo "ITK TEST RESULTS:"
125+
echo "--------------------------------------------------------"
126+
echo "$RESPONSE" | python3 -c "
192127
import sys, json
193128
try:
194129
data = json.load(sys.stdin)
@@ -206,7 +141,8 @@ except Exception as e:
206141
print(f'Raw response: {data if \"data\" in locals() else \"no data\"}')
207142
sys.exit(1)
208143
"
209-
RESULT=$?
144+
RESULT=$?
145+
fi
210146
set -e
211147

212148
if [ $RESULT -ne 0 ]; then

‎itk/scenarios.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"tests": [
3+
{
4+
"name": "Star Topology (Full) - JSONRPC & GRPC",
5+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
6+
"traversal": "euler",
7+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
8+
"protocols": ["jsonrpc", "grpc"],
9+
"behavior": "send_message"
10+
},
11+
{
12+
"name": "Star Topology (No Go v03) - HTTP_JSON",
13+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
14+
"traversal": "euler",
15+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
16+
"protocols": ["http_json"],
17+
"behavior": "send_message"
18+
},
19+
{
20+
"name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
21+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
22+
"traversal": "euler",
23+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
24+
"protocols": ["jsonrpc", "grpc"],
25+
"streaming": true,
26+
"behavior": "send_message"
27+
},
28+
{
29+
"name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
30+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
31+
"traversal": "euler",
32+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
33+
"protocols": ["http_json"],
34+
"streaming": true,
35+
"behavior": "send_message"
36+
},
37+
{
38+
"name": "Push Notification Test - JSONRPC & GRPC",
39+
"sdks": ["current", "python_v10", "python_v03", "go_v03"],
40+
"traversal": "euler",
41+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
42+
"protocols": ["jsonrpc", "grpc"],
43+
"behavior": "push_notification"
44+
},
45+
{
46+
"name": "Push Notification Test - HTTP_JSON",
47+
"sdks": ["current", "python_v10", "python_v03"],
48+
"traversal": "euler",
49+
"edges": ["0->1", "0->2", "1->0", "2->0"],
50+
"protocols": ["http_json"],
51+
"behavior": "push_notification"
52+
},
53+
{
54+
"name": "Resubscribe Test - JSONRPC",
55+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
56+
"traversal": "euler",
57+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
58+
"protocols": ["jsonrpc"],
59+
"streaming": true,
60+
"behavior": "resubscribe"
61+
},
62+
{
63+
"name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
64+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
65+
"traversal": "euler",
66+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
67+
"protocols": ["grpc", "http_json"],
68+
"streaming": true,
69+
"behavior": "resubscribe"
70+
}
71+
]
72+
}

0 commit comments

Comments
 (0)