Transformers v4.12.x compatible (#107)

Update to support Huggingface Transformers v4.12.0 - v4.12.5
Added compatibility with ProphetNet

Author: JulianneKnott

README.md

@@ -16,17 +16,18 @@ Below shows the generation speed gain by using FastSeq.
 
 | Model            | W/O FastSeq (in samples/s) | W/ FastSeq (in samples/s) | Speedup |
 |------------------|:--------------------------:|:-------------------------:|:-----:|
-| [ProphetNet](examples/prophetnet/README.md)       | 2.8 | 11.9  | 4.3  |
+| [ProphetNet (`fs`)](examples/prophetnet/README.md)       | 2.8 | 11.9  | 4.3  |
 | [Bart (`fs`)](examples/bart/README.md)              | 3.3  | 25.1 | 7.7x  |
-| [Bart (`hf`)](examples/bart/README.md#speedup-bart-huggingface-transformers-version-by-using-fastseq) | 2.5 | 12.4 | 5.0x  |
-| [DistilBart (`hf`)](examples/distilbart/README.md)    | 3.4  | 18.5  | 5.4x  |
-| [T5 (`hf`)](examples/t5/README.md)                  | 8.7  | 31.3  | 3.6x  |
+| [Bart (`hf`)](examples/bart/README.md#speedup-bart-huggingface-transformers-version-by-using-fastseq) | 4.5 | 12.4 | 2.8x  |
+| [DistilBart (`hf`)](examples/distilbart/README.md)    | 5.5  | 19.1  | 3.5x  |
+| [T5 (`hf`)](examples/t5/README.md)                  | 9.5  | 31.7  | 3.3x  |
 | [WMT16 En-De (`fs`)](examples/wmt/README.md)        | 144.5   | 422.8  | 2.9x  |
-| [GPT2 (`hf`)](examples/gpt2/README.md)        | 3.0   | 16.7  | 5.5x  |
-| [UniLM (`hf`)](examples/unilm/README.md)        | 1.7   | 16.4  | 9.6x  |
+| [GPT2 (`hf`)](examples/gpt2/README.md)        | 3.9   | 21.8  | 5.6x  |
+| [ProphetNet (`hf`)](examples/prophetnet/README.md)        | 3.4   | 6.2  | 1.8x  |
 
 - All benchmarking experiments run on NVIDIA-V100-16GB with [docker](docker/Dockerfile). Highest speed recorded for each model by tuning batch size. For parameter setting details, click link of corresponding model.
-- `fs` stands for [Fairseq](https://github.com/pytorch/fairseq) 0.10.2 version, `hf` stands for [Huggingface Transformers](https://github.com/huggingface/transformers) 3.0.2 version.
+- The baseline (W/O Fastseq) for [ProphetNet (`fs`)](examples/prophetnet/README.md) is run with fairseq 0.9.0, as it has not yet been updated for compatibility with version 0.10.2
+- `fs` stands for [Fairseq](https://github.com/pytorch/fairseq) 0.10.2 version, `hf` stands for [Huggingface Transformers](https://github.com/huggingface/transformers) 4.12.0 version.
 - Optimizations were automatically applied to all generation/sequence models in Fairseq & Huggingface Transformers. Above only lists a subset of them.
 
 ## How it works?
@@ -39,7 +40,7 @@ FastSeq develops multiple speedup techniques, including an attention cache optim
 - Python version >= 3.6
 - [torch](http://pytorch.org/) >= 1.4.0
 - [fairseq](https://github.com/pytorch/fairseq) >= 0.10.0
-- [transformers](https://github.com/huggingface/transformers) == 3.0.2
+- [transformers](https://github.com/huggingface/transformers) >= 4.12.0
 - [requests](https://pypi.org/project/requests/) >= 2.24.0
 - [absl-py](https://pypi.org/project/absl-py/) >= 0.9.0
 - [rouge-score](https://pypi.org/project/rouge-score/) >= 0.0.4

azure-pipelines.yml

@@ -15,21 +15,21 @@ jobs:
       demands:
         - agent.name -equals gpu3
     container:
-      image: adsbrainwestus2.azurecr.io/fastseq:dev-py3 
-      endpoint: fastseq-acr
+      image: huggingface/transformers-pytorch-gpu:latest
       options: --gpus device=3
     steps:
     - script: |
         #install fastseq
-        which pip
-        which python 
+        pip install --upgrade pip
+        pip install sentencepiece==0.1.96
+        pip install torch==1.10.0
 
         echo "******* Installing fairseq *******"
         pip install fairseq==0.10.2
         pip show fairseq
 
         echo "******* Installing transformers *******"
-        pip install transformers
+        pip install transformers==4.12.0
         pip show transformers
 
         echo "******* Installing fastseq *******"
@@ -39,10 +39,6 @@ jobs:
         echo "******* Adding local bin to path *******"
         export PATH="$HOME/bin:$HOME/.local/bin:$PATH"
 
-        echo "******* Running fastseq unittests *******"
-        pip install pytorch-transformers==1.0.0
-        bash tests/run_fastseq_tests.sh
-
         #cd benchmarks/
         #bash run_all_benchmarks.sh
 
@@ -53,11 +49,16 @@ jobs:
         python -c "import torch; print('torch:', torch.__version__, torch)"
         python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
+        echo "******* Running transformers unittests *******"
+        bash tests/run_transformers_tests.sh
+
         echo "******* Running fairseq unittests *******"
+        pip install apex==0.9.10.dev0
         bash tests/run_fairseq_tests.sh
 
-        echo "******* Running transformers unittests *******"
-        bash tests/run_transformers_tests.sh
+        echo "******* Running fastseq unittests *******"
+        pip install pytorch-transformers==1.0.0
+        bash tests/run_fastseq_tests.sh
 
       displayName: 'run fastseq unit tests'
     - task: PublishTestResults@2

benchmarks/hf.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 source utils.sh
 if [[ $SKIP_BASELINE -eq 0 ]]; then
-    export BASELINE_REPO=$CACHE_DIR/transformers_v3.0.2
-    #https://github.com/huggingface/transformers.git \
+    export BASELINE_REPO=$CACHE_DIR/transformers_v4.12.0
     git_clone_if_not_in_cache \
-	https://github.com/JiushengChen/transformers.git \
+	https://github.com/huggingface/transformers.git \
         $BASELINE_REPO \
-        v3.0.2-ngram
+        v4.12.0
 fi

benchmarks/models/hf_bart.sh

@@ -15,7 +15,7 @@ source hf.sh
     facebook/bart-large-cnn \
     cnn_dm/raw \
     val \
-    32 \
+    32/64 \
     --task summarization \
     --no_repeat_ngram_size 3
 ./benchmark.sh \
@@ -33,16 +33,16 @@ grep "facebook/bart-large-cnn cnn_dm/raw val " perf \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
 	| ./range.sh 0.447 0.448
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
+grep -E "transformers_v4.12.0 facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 2 3
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
+	| ./range.sh 4 5
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 7 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
+	| ./range.sh 10 100
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 11 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 128 " perf \
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 12 100
 

benchmarks/models/hf_distibart.sh

@@ -32,12 +32,12 @@ grep "hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val " perf \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
 	| ./range.sh 0.45 0.452
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
+grep -E "transformers_v4.12.0 hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 3 4
-grep -E "transformers_v3.0.2\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
+	| ./range.sh 5 6
+grep -E "transformers_v4.12.0\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 16.5 100
-grep -E "transformers_v3.0.2\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
+	| ./range.sh 17 100
+grep -E "transformers_v4.12.0\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 18.3 100
+	| ./range.sh 18 100

benchmarks/models/hf_gpt2.sh

@@ -7,7 +7,6 @@
 #   <split> # train/val/test (text) or train/valid/test (binary)
 #   <batch-sizes>
 source hf.sh
-
 # MODEL - bart large cnn from transformer
 # TASK - cnn dm val full set
 
@@ -16,7 +15,7 @@ source hf.sh
     gpt2 \
     cnn_dm/raw \
     val \
-    64/128 \
+    64/128/256 \
     --task summarization \
     --no_repeat_ngram_size 3 \
     --max_tokenizer_length 512 \
@@ -27,7 +26,7 @@ source hf.sh
     gpt2 \
     cnn_dm/raw \
     val \
-    64 \
+    64/128 \
     --task summarization \
     --no_repeat_ngram_size 3 \
     --max_tokenizer_length 512 \
@@ -37,14 +36,18 @@ source hf.sh
 grep "gpt2 cnn_dm/raw val " perf \
 	| awk '{print $9}' \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
-	| ./range.sh 0.155 0.156
+	| ./range.sh 0.160 0.162
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 gpt2 cnn_dm/raw val 64 " perf \
+grep -E "transformers_v4.12.0 gpt2 cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 2.9 3.2
-grep -E "transformers_v3.0.2\+fastseq_v.* gpt2 cnn_dm/raw val 64 " perf \
+	| ./range.sh 3.5 4.5
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 64 " perf \
+	| awk '{s+=$13}END{print s/NR}' \
+	| ./range.sh 16 100
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 10.8 11.3
-grep -E "transformers_v3.0.2\+fastseq_v.* gpt2 cnn_dm/raw val 128 " perf \
+	| ./range.sh 20 100
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 256 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 16.4 16.8
+	| ./range.sh 21 100
+    

benchmarks/models/hf_mbart.sh

@@ -26,11 +26,11 @@ source hf.sh
 # Accuracy
 grep "facebook/mbart-large-en-ro wmt_en_ro/raw val " perf \
 	| awk '{if($8!="NA"){c+=1;s+=$8}}END{print s/c}' \
-	| ./range.sh 56.1 56.3
+	| ./range.sh 56.1 56.4
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0 facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
 	| ./range.sh 6.0 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 9 100

benchmarks/models/hf_prophetnet.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Run it at its parent folder, and check result at ../perf.
+# USAGE - ./benchmark.sh
+#   [fairseq|fairseq+fastseq|transformers|transformers+fastseq]
+#   <model>
+#   <task>
+#   <split> # train/val/test (text) or train/valid/test (binary)
+#   <batch-sizes>
+source hf.sh
+
+# MODEL - prophetnet from transformer
+# TASK - cnn dm val full set
+./benchmark.sh \
+    transformers \
+    microsoft/prophetnet-large-uncased \
+    cnn_dm_bert/raw \
+    val \
+    128 \
+    --task summarization \
+    --no_repeat_ngram_size 3
+./benchmark.sh \
+    transformers+fastseq \
+    microsoft/prophetnet-large-uncased \
+    cnn_dm_bert/raw \
+    val \
+    128 \
+    --task summarization \
+    --no_repeat_ngram_size 3
+
+# Accuracy
+grep "microsoft/prophetnet-large-uncased cnn_dm_bert/raw val " perf \
+	| awk '{print $9}' \
+	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
+	| ./range.sh 0.230 0.232
+# Speed on V100 16GB 250W
+grep -E "transformers_v4.12.0 microsoft/prophetnet-large-uncased cnn_dm_bert/raw val 128 " perf \
+	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
+	| ./range.sh 3 4
+grep -E "transformers_v4.12.0+fastseq_v.* microsoft/prophetnet-large-uncased cnn_dm_bert/raw val 128 " perf \
+	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
+	| ./range.sh 6 100

benchmarks/models/hf_t5.sh

@@ -14,28 +14,30 @@ source hf.sh
     wmt_en_ro/raw \
     val \
     64 \
-    --task translation_en_to_ro 
-#    --no_repeat_ngram_size 3	# baseline don't support this arg now.
+    --task translation_en_to_ro \
+    --no_repeat_ngram_size 3
+
 ./benchmark.sh \
     transformers+fastseq \
     t5-base \
     wmt_en_ro/raw \
     val \
     64/128 \
     --task translation_en_to_ro \
-    --postprocess_workers 3
-#    --no_repeat_ngram_size 3
-# Accuracy
+    --postprocess_workers 3 \
+    --no_repeat_ngram_size 3
+
+# # Accuracy
 grep "t5-base wmt_en_ro/raw val " perf \
 	| awk '{if($8!="NA"){c+=1;s+=$8}}END{print s/c}' \
-	| ./range.sh 57.8 57.9
+	| ./range.sh 58.0 59.0
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 t5-base wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0 t5-base wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 8 10
-grep -E "transformers_v3.0.2\+fastseq_v.* t5-base wmt_en_ro/raw val 64 " perf \
+	| ./range.sh 12 17
+grep -E "transformers_v4.12.0\+fastseq_v.* t5-base wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 19 100
-grep -E "transformers_v3.0.2\+fastseq_v.* t5-base wmt_en_ro/raw val 128 " perf \
+	| ./range.sh 23 100
+grep -E "transformers_v4.12.0\+fastseq_v.* t5-base wmt_en_ro/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 30 100