Skip to content

[CI] Modify test workflow #1805

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions .github/actions/inductor-xpu-e2e-test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,20 @@ runs:
cd benchmark && git checkout $TORCHBENCH_COMMIT_ID
# remove deps which will reinstall torch
pip install --no-deps accelerate
pip install --no-deps $(cat requirements.txt |grep 'pytorch-image-models')
timm_commit="$(grep 'pytorch-image-models' requirements.txt |awk -F '@' '{print $2}')"
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/${timm_commit:-"main"}/requirements.txt | grep -vE torch)
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g' requirements.txt
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
pip install -U transformers==4.44.2
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g' requirements.txt
git status && git diff
pip install -r requirements.txt
python install.py --continue_on_fail
# deps for torchrec_dlrm
pip install pyre_extensions
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
fi
if [[ ${{ inputs.suite }} == *"huggingface"* ]]; then
pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
pip install -U transformers==4.44.2
fi
if [[ ${{ inputs.suite }} == *"timm_models"* ]]; then
if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
Expand Down
75 changes: 42 additions & 33 deletions .github/actions/pt2e/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,20 @@ runs:
fi
# torchbench
python -c "import torch, torchvision, torchaudio"
cd pt2e-performance && pip install -r requirements.txt
cd pt2e-performance
# remove deps which will reinstall torch
pip install --no-deps accelerate
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
pip install -U transformers==4.44.2
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g' requirements.txt
git status && git diff
pip install -r requirements.txt
python install.py --continue_on_fail
cd ../
# deps for torchrec_dlrm
pip install pyre_extensions
pip install fbgemm-gpu
pip install --no-deps torchmetrics==1.0.3 torchrec
# transformers
pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
# timm
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
fi
pip install numpy==1.26.4
# dataset
Expand All @@ -97,35 +99,42 @@ runs:
pt2e_logs_dir="${{ github.workspace }}/../pytorch/inductor_log/pt2e"
rm -rf "${pt2e_logs_dir}" && mkdir -p "${pt2e_logs_dir}"
if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet |\
tee "${pt2e_logs_dir}/accuracy-fp32.log"
fi
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet |\
tee "${pt2e_logs_dir}/accuracy-int8.log"
fi
models="alexnet,mnasnet1_0,mobilenet_v2,mobilenet_v3_large,resnet152,resnet18,resnet50,resnext50_32x4d,shufflenet_v2_x1_0,squeezenet1_1,vgg16"
for model_name in $(echo $models |sed 's/,/ /g')
do
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
tee "${pt2e_logs_dir}/accuracy-fp32-${model_name}.log"
fi
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
tee "${pt2e_logs_dir}/accuracy-int8-${model_name}.log"
fi
done
fi
if [[ "${{ inputs.scenario }}" == *"performance"* ]];then
models="alexnet,demucs,dlrm,hf_Albert,hf_Bert,hf_Bert_large,hf_DistilBert,hf_Roberta_base,mnasnet1_0,mobilenet_v2,"
models+="mobilenet_v3_large,nvidia_deeprecommender,pytorch_CycleGAN_and_pix2pix,resnet152,resnet18,resnet50,resnext50_32x4d,"
models+="shufflenet_v2_x1_0,squeezenet1_1,Super_SloMo,timm_efficientnet,timm_nfnet,timm_regnet,timm_resnest,"
models+="timm_vision_transformer,timm_vision_transformer_large,timm_vovnet,vgg16"
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
rm -rf pt2e-performance/.userbenchmark
python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor -m $models 2>&1 |\
tee "${pt2e_logs_dir}/performance-fp32.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32
fi
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
rm -rf pt2e-performance/.userbenchmark
XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM
for model_name in $(echo $models |sed 's/,/ /g')
do
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
rm -rf pt2e-performance/.userbenchmark
python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
--torchdynamo inductor -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-fp32.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32-${model_name}
fi
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
rm -rf pt2e-performance/.userbenchmark
XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
--torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM-${model_name}

rm -rf pt2e-performance/.userbenchmark
XPU_QUANT_CONFIG=SYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM
fi
rm -rf pt2e-performance/.userbenchmark
XPU_QUANT_CONFIG=SYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
--torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM-${model_name}
fi
done
fi
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accura
detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,fail_accuracy
detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,pass
detectron2_fcos_r_50_fpn,pass,pass,pass,pass,pass
detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run,fail_to_run
detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
# https://github.com/intel/torch-xpu-ops/issues/1796
detectron2_maskrcnn_r_50_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
detectron2_maskrcnn_r_50_fpn,eager_1st_run_OOM,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
dlrm,pass,pass,pass,pass,pass
doctr_det_predictor,pass,pass,pass,pass,pass
doctr_reco_predictor,pass,pass,pass,pass,pass
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ drq,pass,pass,pass,pass,pass
fastNLP_Bert,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/508
functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass
functorch_maml_omniglot,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1807
functorch_maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
hf_Albert,pass,pass,pass,pass,pass
hf_Bart,pass,pass,pass,pass,pass
hf_Bert,pass,pass,pass,pass,pass
Expand All @@ -56,19 +57,22 @@ llama,pass,pass,pass,pass,pass
llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
maml_omniglot,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1807
maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass
mnasnet1_0,pass,pass,pass,pass,pass
mobilenet_v2,pass,pass,pass,pass,pass
mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
# https://github.com/intel/torch-xpu-ops/issues/1787
mobilenet_v2_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
mobilenet_v3_large,pass,pass,pass,pass,pass
moco,model_fail_to_load,model_fail_to_load,eager_fail_to_run,model_fail_to_load,model_fail_to_load
moondream,pass,pass,pass,pass,pass
nanogpt,pass,pass,pass,pass,pass
nvidia_deeprecommender,pass,pass,pass,pass,pass
opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
phlippe_densenet,pass,pass,pass,pass,pass
phlippe_resnet,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1799
phlippe_resnet,pass,fail_accuracy,pass,pass,pass
pyhpc_equation_of_state,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
pyhpc_isoneutral_mixing,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
Expand All @@ -78,14 +82,16 @@ pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
resnet152,pass,pass,pass,pass,pass
resnet18,pass,pass,pass,pass,pass
resnet50,pass,pass,pass,pass,pass
resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
# https://github.com/intel/torch-xpu-ops/issues/1787
resnet50_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
resnext50_32x4d,pass,pass,pass,pass,pass
sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
shufflenet_v2_x1_0,pass,pass,pass,pass,pass
simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
soft_actor_critic,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1806
soft_actor_critic,pass,pass,pass,pass,eager_fail_to_run
speech_transformer,pass,pass,pass,pass,pass
squeezenet1_1,pass,pass,pass,pass,pass
stable_diffusion_text_encoder,pass,pass,pass,pass,pass
Expand All @@ -96,7 +102,7 @@ timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model
timm_efficientnet,pass,pass,pass,pass,pass
timm_nfnet,pass,pass,pass,pass,pass
# https://github.com/intel/torch-xpu-ops/issues/1334
timm_regnet,pass,fail_accuracy,pass,pass,pass
timm_regnet,pass,fail_accuracy,fail_accuracy,pass,pass
timm_resnest,pass,pass,pass,pass,pass
timm_vision_transformer,pass,pass,pass,pass,pass
timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/_linux_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@ on:
type: string
default: 'linux.idc.xpu'
description: Runner label
update_lkg:
required: false
type: string
default: 'false'
description: Whether update LKG torch version to issue #1280
outputs:
torch_commit_id:
description: The commit id of the torch build
Expand Down
Loading