Skip to content

[CD] Update nightly test workflow #1941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[CD] Update nightly test workflow #1941

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a002786
modify torchbench commit pinned path from pytorch
mengfei25 Aug 19, 2025
8f9c5fa
fix pt2e accuracy test block issue
mengfei25 Aug 19, 2025
29c4133
update accuracy
mengfei25 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/actions/pt2e/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,14 @@ runs:
do
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
${cmd_line} --model_list ${model_name} --is_fp32 2>&1 |tee "${pt2e_logs_dir}/accuracy-float32-${model_name}.log" || true
grep -i 'Acc.1.*Acc.5' "${pt2e_logs_dir}/accuracy-float32-${model_name}.log" |tail -n 1 |awk -v m="${model_name}" '
(grep -i 'Acc.1.*Acc.5' "${pt2e_logs_dir}/accuracy-float32-${model_name}.log" || echo "failed & failed") 2>&1 |tail -n 1 |awk -v m="${model_name}" '
BEGIN{acc1 = "failed"; acc5 = "failed";}
{acc1 = $(NF - 2); acc5 = $NF;}
END{printf("Accuracy,%s,float32,%s,%s\n", m, acc1, acc5) }' |tee -a ${pt2e_logs_dir}/summary.csv
fi
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
${cmd_line} --model_list ${model_name} 2>&1 |tee "${pt2e_logs_dir}/accuracy-int8-${model_name}.log" || true
grep -i 'Acc.1.*Acc.5' "${pt2e_logs_dir}/accuracy-int8-${model_name}.log" |tail -n 1 |awk -v m="${model_name}" '
(grep -i 'Acc.1.*Acc.5' "${pt2e_logs_dir}/accuracy-int8-${model_name}.log" || echo "failed & failed") 2>&1 |tail -n 1 |awk -v m="${model_name}" '
BEGIN{acc1 = "failed"; acc5 = "failed";}
{acc1 = $(NF - 2); acc5 = $NF;}
END{printf("Accuracy,%s,int8,%s,%s\n", m, acc1, acc5) }' |tee -a ${pt2e_logs_dir}/summary.csv
Expand Down
4 changes: 2 additions & 2 deletions .github/ci_expected_accuracy/rolling/inductor_timm_models_training.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ botnet26t_256,pass,pass,pass,pass,pass
cait_m36_384,pass,pass,fail_accuracy,pass,pass
coat_lite_mini,pass,pass,pass,pass,pass
convit_base,pass,pass,pass,pass,pass
convmixer_768_32,pass,fail_accuracy,pass,fail_accuracy,pass
convmixer_768_32,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1274
convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass
crossvit_9_240,pass,pass,pass,pass,pass
Expand Down Expand Up @@ -52,7 +52,7 @@ sebotnet33ts_256,pass,pass,pass,pass,pass
selecsls42b,pass,pass,pass,pass,pass
spnasnet_100,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1768
swin_base_patch4_window7_224,pass,pass,pass,pass,pass
swin_base_patch4_window7_224,pass,fail_accuracy,fail_accuracy,pass,pass
swsl_resnext101_32x16d,pass,pass,pass,pass,pass
tf_efficientnet_b0,pass,pass,pass,pass,pass
tf_mixnet_l,pass,pass,pass,pass,pass
Expand Down
6 changes: 3 additions & 3 deletions .github/ci_expected_accuracy/rolling/inductor_torchbench_inference.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accura
detectron2_fasterrcnn_r_101_fpn,eager_1st_run_OOM,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_fpn,eager_1st_run_OOM,eager_fail_to_run,eager_1st_run_OOM,fail_accuracy,pass
detectron2_fasterrcnn_r_50_fpn,eager_1st_run_OOM,eager_fail_to_run,eager_1st_run_OOM,fail_accuracy,eager_1st_run_OOM
detectron2_fcos_r_50_fpn,pass,pass,pass,fail_accuracy,pass
detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run,fail_to_run
detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
Expand Down Expand Up @@ -92,10 +92,10 @@ simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_t
simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
soft_actor_critic,pass,pass,pass,pass,pass
speech_transformer,pass,pass,pass,pass,pass
squeezenet1_1,pass,fail_accuracy,pass,pass,pass
squeezenet1_1,pass,pass,pass,pass,pass
stable_diffusion_text_encoder,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1261
stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run
stable_diffusion_unet,eager_fail_to_run,pass_due_to_skip,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run
tacotron2,pass,pass,pass,fail_to_run,fail_to_run
timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
timm_efficientnet,pass,pass,pass,pass,pass
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly_ondemand.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ jobs:
fi
echo "TORCH_BRANCH_ID=${{ inputs.pytorch }}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCH_COMMIT_ID=$(git rev-parse HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.github/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.ci/docker/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHVISION_COMMIT_ID=$(<.github/ci_commit_pins/vision.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHAUDIO_COMMIT_ID=$(<.github/ci_commit_pins/audio.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly_ondemand_rolling.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ jobs:
fi
echo "TORCH_BRANCH_ID=${{ inputs.pytorch }}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCH_COMMIT_ID=$(git rev-parse HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.github/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.ci/docker/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHVISION_COMMIT_ID=$(<.github/ci_commit_pins/vision.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHAUDIO_COMMIT_ID=$(<.github/ci_commit_pins/audio.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly_ondemand_whl.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ jobs:
echo "TORCHAUDIO_COMMIT_ID=$(python -c 'import torchaudio; print(torchaudio.version.git_version)')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TRITON_COMMIT_ID=$(python -c 'import triton; print(triton.__version__)')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
cd ../pytorch
echo "TORCHBENCH_COMMIT_ID=$(<.github/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.ci/docker/ci_commit_pins/torchbench.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TIMM_COMMIT_ID=$(<.ci/docker/ci_commit_pins/timm.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "MODEL_ONLY_NAME=${{ inputs.model }}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pull.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ jobs:
echo "TORCH_COMMIT_ID=$(git rev-parse HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
echo "TRITON_COMMIT_ID=$(<.ci/docker/ci_commit_pins/triton-xpu.txt)" >> "${GITHUB_ENV}"
echo "TORCHVISION_COMMIT_ID=$(<.github/ci_commit_pins/vision.txt)" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.github/ci_commit_pins/torchbench.txt)" >> "${GITHUB_ENV}"
echo "TORCHBENCH_COMMIT_ID=$(<.ci/docker/ci_commit_pins/torchbench.txt)" >> "${GITHUB_ENV}"
echo "TORCHAUDIO_COMMIT_ID=$(<.github/ci_commit_pins/audio.txt)" >> "${GITHUB_ENV}"
echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" >> "${GITHUB_ENV}"
echo "TIMM_COMMIT_ID=$(<.ci/docker/ci_commit_pins/timm.txt)" >> "${GITHUB_ENV}"
Expand Down
Loading