[CI] Modify test workflow

.github/actions/inductor-xpu-e2e-test/action.yml

@@ -65,19 +65,20 @@ runs:
           cd benchmark && git checkout $TORCHBENCH_COMMIT_ID
           # remove deps which will reinstall torch
           pip install --no-deps accelerate
-          pip install --no-deps $(cat requirements.txt |grep 'pytorch-image-models')
-          timm_commit="$(grep 'pytorch-image-models' requirements.txt  |awk -F '@' '{print $2}')"
-          pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/${timm_commit:-"main"}/requirements.txt | grep -vE torch)
-          sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g'  requirements.txt
+          pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
+          pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
+          pip install -U transformers==4.44.2
+          sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g'  requirements.txt
+          git status && git diff
           pip install -r requirements.txt
           python install.py --continue_on_fail
           # deps for torchrec_dlrm
           pip install pyre_extensions
-          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
+          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
           pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
         fi
         if [[ ${{ inputs.suite }} == *"huggingface"* ]]; then
-          pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
+          pip install -U transformers==4.44.2
         fi
         if [[ ${{ inputs.suite }} == *"timm_models"* ]]; then
           if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then

.github/actions/pt2e/action.yml

@@ -62,18 +62,20 @@ runs:
           fi
           # torchbench
           python -c "import torch, torchvision, torchaudio"
-          cd pt2e-performance && pip install -r requirements.txt
+          cd pt2e-performance
+          # remove deps which will reinstall torch
+          pip install --no-deps accelerate
+          pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
+          pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
+          pip install -U transformers==4.44.2
+          sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g'  requirements.txt
+          git status && git diff
+          pip install -r requirements.txt
           python install.py --continue_on_fail
-          cd ../
           # deps for torchrec_dlrm
           pip install pyre_extensions
-          pip install fbgemm-gpu
-          pip install --no-deps torchmetrics==1.0.3 torchrec
-          # transformers
-          pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
-          # timm
-          pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
-          pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
+          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
+          pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
         fi
         pip install numpy==1.26.4
         # dataset
@@ -97,35 +99,42 @@ runs:
         pt2e_logs_dir="${{ github.workspace }}/../pytorch/inductor_log/pt2e"
         rm -rf "${pt2e_logs_dir}" && mkdir -p "${pt2e_logs_dir}"
         if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then
-          if [[ "${{ inputs.dt }}" == *"float32"* ]];then
-            python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet |\
-                    tee "${pt2e_logs_dir}/accuracy-fp32.log"
-          fi
-          if [[ "${{ inputs.dt }}" == *"int8"* ]];then
-            python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet |\
-                    tee "${pt2e_logs_dir}/accuracy-int8.log"
-          fi
+          models="alexnet,mnasnet1_0,mobilenet_v2,mobilenet_v3_large,resnet152,resnet18,resnet50,resnext50_32x4d,shufflenet_v2_x1_0,squeezenet1_1,vgg16"
+          for model_name in $(echo $models |sed 's/,/ /g')
+          do
+            if [[ "${{ inputs.dt }}" == *"float32"* ]];then
+              python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
+                      tee "${pt2e_logs_dir}/accuracy-fp32-${model_name}.log"
+            fi
+            if [[ "${{ inputs.dt }}" == *"int8"* ]];then
+              python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
+                      tee "${pt2e_logs_dir}/accuracy-int8-${model_name}.log"
+            fi
+          done
         fi
         if [[ "${{ inputs.scenario }}" == *"performance"* ]];then
           models="alexnet,demucs,dlrm,hf_Albert,hf_Bert,hf_Bert_large,hf_DistilBert,hf_Roberta_base,mnasnet1_0,mobilenet_v2,"
           models+="mobilenet_v3_large,nvidia_deeprecommender,pytorch_CycleGAN_and_pix2pix,resnet152,resnet18,resnet50,resnext50_32x4d,"
           models+="shufflenet_v2_x1_0,squeezenet1_1,Super_SloMo,timm_efficientnet,timm_nfnet,timm_regnet,timm_resnest,"
           models+="timm_vision_transformer,timm_vision_transformer_large,timm_vovnet,vgg16"
-          if [[ "${{ inputs.dt }}" == *"float32"* ]];then
-            rm -rf pt2e-performance/.userbenchmark
-            python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor -m $models 2>&1 |\
-                    tee "${pt2e_logs_dir}/performance-fp32.log"
-            mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32
-          fi
-          if [[ "${{ inputs.dt }}" == *"int8"* ]];then
-            rm -rf pt2e-performance/.userbenchmark
-            XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
-                    tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
-            mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM
+          for model_name in $(echo $models |sed 's/,/ /g')
+          do
+            if [[ "${{ inputs.dt }}" == *"float32"* ]];then
+              rm -rf pt2e-performance/.userbenchmark
+              python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
+                      --torchdynamo inductor -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-fp32.log"
+              mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32-${model_name}
+            fi
+            if [[ "${{ inputs.dt }}" == *"int8"* ]];then
+              rm -rf pt2e-performance/.userbenchmark
+              XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
+                      --torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
+              mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM-${model_name}
 
-            rm -rf pt2e-performance/.userbenchmark
-            XPU_QUANT_CONFIG=SYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
-                                tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
-            mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM
-          fi
+              rm -rf pt2e-performance/.userbenchmark
+              XPU_QUANT_CONFIG=SYMM  python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
+                      --torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
+              mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM-${model_name}
+            fi
+          done
         fi

.github/ci_expected_accuracy/rolling/inductor_torchbench_inference.csv

@@ -22,13 +22,14 @@ detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accura
 detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
 detectron2_fasterrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
 detectron2_fasterrcnn_r_50_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
-detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,fail_accuracy
+detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,pass
 detectron2_fcos_r_50_fpn,pass,pass,pass,pass,pass
 detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run,fail_to_run
 detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
 detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
-detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
-detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
+# https://github.com/intel/torch-xpu-ops/issues/1796
+detectron2_maskrcnn_r_50_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
+detectron2_maskrcnn_r_50_fpn,eager_1st_run_OOM,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
 dlrm,pass,pass,pass,pass,pass
 doctr_det_predictor,pass,pass,pass,pass,pass
 doctr_reco_predictor,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/rolling/inductor_torchbench_training.csv

@@ -33,7 +33,8 @@ drq,pass,pass,pass,pass,pass
 fastNLP_Bert,pass,pass,pass,pass,pass
 # https://github.com/intel/torch-xpu-ops/issues/508
 functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass
-functorch_maml_omniglot,pass,pass,pass,pass,pass
+# https://github.com/intel/torch-xpu-ops/issues/1807
+functorch_maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
 hf_Albert,pass,pass,pass,pass,pass
 hf_Bart,pass,pass,pass,pass,pass
 hf_Bert,pass,pass,pass,pass,pass
@@ -56,19 +57,22 @@ llama,pass,pass,pass,pass,pass
 llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
 llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
-maml_omniglot,pass,pass,pass,pass,pass
+# https://github.com/intel/torch-xpu-ops/issues/1807
+maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
 microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass
 mnasnet1_0,pass,pass,pass,pass,pass
 mobilenet_v2,pass,pass,pass,pass,pass
-mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+# https://github.com/intel/torch-xpu-ops/issues/1787
+mobilenet_v2_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 mobilenet_v3_large,pass,pass,pass,pass,pass
 moco,model_fail_to_load,model_fail_to_load,eager_fail_to_run,model_fail_to_load,model_fail_to_load
 moondream,pass,pass,pass,pass,pass
 nanogpt,pass,pass,pass,pass,pass
 nvidia_deeprecommender,pass,pass,pass,pass,pass
 opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 phlippe_densenet,pass,pass,pass,pass,pass
-phlippe_resnet,pass,pass,pass,pass,pass
+# https://github.com/intel/torch-xpu-ops/issues/1799
+phlippe_resnet,pass,fail_accuracy,pass,pass,pass
 pyhpc_equation_of_state,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 pyhpc_isoneutral_mixing,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
@@ -78,14 +82,16 @@ pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
 resnet152,pass,pass,pass,pass,pass
 resnet18,pass,pass,pass,pass,pass
 resnet50,pass,pass,pass,pass,pass
-resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+# https://github.com/intel/torch-xpu-ops/issues/1787
+resnet50_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 resnext50_32x4d,pass,pass,pass,pass,pass
 sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 shufflenet_v2_x1_0,pass,pass,pass,pass,pass
 simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
-soft_actor_critic,pass,pass,pass,pass,pass
+# https://github.com/intel/torch-xpu-ops/issues/1806
+soft_actor_critic,pass,pass,pass,pass,eager_fail_to_run
 speech_transformer,pass,pass,pass,pass,pass
 squeezenet1_1,pass,pass,pass,pass,pass
 stable_diffusion_text_encoder,pass,pass,pass,pass,pass
@@ -96,7 +102,7 @@ timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model
 timm_efficientnet,pass,pass,pass,pass,pass
 timm_nfnet,pass,pass,pass,pass,pass
 # https://github.com/intel/torch-xpu-ops/issues/1334
-timm_regnet,pass,fail_accuracy,pass,pass,pass
+timm_regnet,pass,fail_accuracy,fail_accuracy,pass,pass
 timm_resnest,pass,pass,pass,pass,pass
 timm_vision_transformer,pass,pass,pass,pass,pass
 timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip

.github/workflows/_linux_build.yml

@@ -28,11 +28,6 @@ on:
         type: string
         default: 'linux.idc.xpu'
         description: Runner label
-      update_lkg:
-        required: false
-        type: string
-        default: 'false'
-        description: Whether update LKG torch version to issue #1280
     outputs:
       torch_commit_id:
         description: The commit id of the torch build