Skip to content

Commit 0897d5a

Browse files
Pigbibicodex
Pigbibi
and
codex
authored
fix: restore ibkr gateway watchers on deploy failure (#83)
Co-authored-by: Codex <noreply@openai.com>
1 parent 191f33a commit 0897d5a

4 files changed

Lines changed: 47 additions & 5 deletions

File tree

‎.github/workflows/main.yml‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,20 @@ jobs:
611611
fi
612612
compose_image="\${compose_project_name}-\${compose_service_name}:latest"
613613
resolve_ibkr_gateway_unit_names "\${container_name}" "\${IB_GATEWAY_UNIT_SUFFIX:-}"
614+
watchers_restored=false
615+
restore_gateway_watchers() {
616+
set +e
617+
restore_status=0
618+
echo "[\$(date -u +%FT%TZ)] Restoring gateway watchers"
619+
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh || restore_status="\$?"
620+
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}' || restore_status="\$?"
621+
if [ "\${restore_status}" -eq 0 ]; then
622+
watchers_restored=true
623+
fi
624+
set -e
625+
return "\${restore_status}"
626+
}
627+
trap 'status=\$?; if [ "\${watchers_restored}" != "true" ]; then restore_gateway_watchers || true; fi; exit "\${status}"' EXIT
614628
sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true
615629
sudo systemctl stop "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" "\${IBKR_GATEWAY_DAILY_RESTART_SERVICE}" 2>/dev/null || true
616630
sudo bash ./scripts/ensure_host_swap.sh
@@ -620,7 +634,7 @@ jobs:
620634
fi
621635
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh
622636
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}'
623-
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}'
637+
restore_gateway_watchers
624638
sudo docker compose ps
625639
sudo systemctl status "\${IBKR_2FA_BOT_TIMER}" --no-pager
626640
sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager
@@ -640,6 +654,20 @@ jobs:
640654
container_name="\${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}"
641655
compose_service_name="\${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}"
642656
resolve_ibkr_gateway_unit_names "\${container_name}" "\${IB_GATEWAY_UNIT_SUFFIX:-}"
657+
watchers_restored=false
658+
restore_gateway_watchers() {
659+
set +e
660+
restore_status=0
661+
echo "[\$(date -u +%FT%TZ)] Restoring gateway watchers"
662+
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh || restore_status="\$?"
663+
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}' || restore_status="\$?"
664+
if [ "\${restore_status}" -eq 0 ]; then
665+
watchers_restored=true
666+
fi
667+
set -e
668+
return "\${restore_status}"
669+
}
670+
trap 'status=\$?; if [ "\${watchers_restored}" != "true" ]; then restore_gateway_watchers || true; fi; exit "\${status}"' EXIT
643671
sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true
644672
sudo systemctl stop "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" "\${IBKR_GATEWAY_DAILY_RESTART_SERVICE}" 2>/dev/null || true
645673
sudo bash ./scripts/ensure_host_swap.sh
@@ -649,7 +677,7 @@ jobs:
649677
650678
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh
651679
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}'
652-
sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}'
680+
restore_gateway_watchers
653681
sudo docker compose ps
654682
sudo systemctl status "\${IBKR_2FA_BOT_TIMER}" --no-pager
655683
sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager

‎scripts/recover_ib_gateway_ready.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ set -eu
5353
progress_window_seconds="$1"
5454
progress_regex="$2"
5555
now="$(date +%s)"
56+
cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"
5657
5758
for log_path in /home/ibgateway/Jts/launcher.log /home/ibgateway/2fa.log; do
5859
if [ ! -f "${log_path}" ]; then
@@ -61,7 +62,11 @@ for log_path in /home/ibgateway/Jts/launcher.log /home/ibgateway/2fa.log; do
6162
6263
log_mtime="$(stat -c %Y "${log_path}" 2>/dev/null || echo 0)"
6364
if [ $((now - log_mtime)) -le "${progress_window_seconds}" ]; then
64-
tail -n 400 "${log_path}" 2>/dev/null | grep -Eiq "${progress_regex}" && exit 0
65+
tail -n 400 "${log_path}" 2>/dev/null \
66+
| awk -v cutoff_timestamp="${cutoff_timestamp}" -v progress_regex="${progress_regex}" '
67+
substr($0, 1, 19) >= cutoff_timestamp && $0 ~ progress_regex { found = 1 }
68+
END { exit found ? 0 : 1 }
69+
' && exit 0
6570
fi
6671
done
6772

‎tests/test_gateway_recovery_scripts.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ grep -Fq 'gateway_recently_progressing_from_file_logs()' "$recover_script"
3333
grep -Fq '/home/ibgateway/Jts/launcher.log' "$recover_script"
3434
grep -Fq '/home/ibgateway/2fa.log' "$recover_script"
3535
grep -Fq 'stat -c %Y "${log_path}"' "$recover_script"
36+
grep -Fq 'cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"' "$recover_script"
37+
grep -Fq 'substr($0, 1, 19) >= cutoff_timestamp && $0 ~ progress_regex' "$recover_script"
3638
grep -Fq 'wait_for_ready_with_progress()' "$recover_script"
3739
grep -Fq 'Recent IB gateway login/config progress detected' "$recover_script"
3840
grep -Fq 'IB_GATEWAY_RECOVERY_LOCK_FILE:-/var/lock/ib_gateway_recovery.lock' "$recover_script"

‎tests/test_workflow_shared_config.sh‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ grep -Fq 'sudo bash ./scripts/ensure_host_swap.sh' "$workflow_file"
7474
grep -Fq 'resolve_ibkr_gateway_unit_names "\${container_name}" "\${IB_GATEWAY_UNIT_SUFFIX:-}"' "$workflow_file"
7575
grep -Fq 'sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true' "$workflow_file"
7676
grep -Fq 'bash ./scripts/install_gateway_health_watcher.sh' "$workflow_file"
77+
grep -Fq 'restore_gateway_watchers()' "$workflow_file"
78+
grep -Fq "trap 'status=\\\$?; if [ \"\\\${watchers_restored}\" != \"true\" ]; then restore_gateway_watchers || true; fi; exit \"\\\${status}\"' EXIT" "$workflow_file"
7779
grep -Fq "sudo env IB_GATEWAY_CONTAINER_NAME=\"\\\${container_name}\" IB_GATEWAY_COMPOSE_SERVICE_NAME=\"\\\${compose_service_name}\" bash ./scripts/recover_ib_gateway_ready.sh '\${IB_GATEWAY_MODE}'" "$workflow_file"
7880
grep -Fq 'sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager' "$workflow_file"
7981
grep -Fq 'sudo systemctl status "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" --no-pager' "$workflow_file"
@@ -88,11 +90,16 @@ health_watcher_lines=()
8890
while IFS= read -r line_number; do
8991
health_watcher_lines+=("$line_number")
9092
done < <(grep -nF 'bash ./scripts/install_gateway_health_watcher.sh' "$workflow_file" | cut -d: -f1)
93+
restore_call_lines=()
94+
while IFS= read -r line_number; do
95+
restore_call_lines+=("$line_number")
96+
done < <(grep -nFx ' restore_gateway_watchers' "$workflow_file" | cut -d: -f1)
9197
test "${#recover_lines[@]}" -eq 2
9298
test "${#health_watcher_lines[@]}" -eq 2
99+
test "${#restore_call_lines[@]}" -eq 2
93100
for i in 0 1; do
94-
if [ "${recover_lines[$i]}" -ge "${health_watcher_lines[$i]}" ]; then
95-
echo "Gateway health watcher must be installed after explicit recovery in deploy block $i" >&2
101+
if [ "${recover_lines[$i]}" -ge "${restore_call_lines[$i]}" ]; then
102+
echo "Gateway watchers must be restored after explicit recovery in deploy block $i" >&2
96103
exit 1
97104
fi
98105
done

0 commit comments

Comments
 (0)