feat: add encoder_fairseq_dir path to fine-tuning and inference scripts

Author: cwx-worst-one

examples/slam_aac/scripts/finetune_audiocaps.sh

@@ -9,6 +9,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 
@@ -38,6 +40,7 @@ hydra.run.dir=$output_dir \
 ++model_config.encoder_path=$audio_encoder_path \
 ++model_config.encoder_dim=768 \
 ++model_config.encoder_projector=linear \
+++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
 ++dataset_config.encoder_projector_ds_rate=${encoder_projector_ds_rate} \
 ++dataset_config.dataset=audio_dataset \
 ++dataset_config.train_data_path=$train_jsonl_path \

examples/slam_aac/scripts/finetune_clotho.sh

@@ -9,6 +9,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 
@@ -38,6 +40,7 @@ hydra.run.dir=$output_dir \
 ++model_config.encoder_path=$audio_encoder_path \
 ++model_config.encoder_dim=768 \
 ++model_config.encoder_projector=linear \
+++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
 ++dataset_config.encoder_projector_ds_rate=${encoder_projector_ds_rate} \
 ++dataset_config.dataset=audio_dataset \
 ++dataset_config.train_data_path=$train_jsonl_path \

examples/slam_aac/scripts/inference_audiocaps_CLAP_Refine.sh

@@ -10,6 +10,8 @@ audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 clap_dir=/data/xiquan.li/models/clap
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 encoder_projector_ds_rate=5
 
 inference_data_path=/data/wenxi.chen/data/audiocaps/new_test.jsonl
@@ -41,6 +43,7 @@ for num_beams in "${beam_range[@]}"; do
         ++model_config.encoder_projector=linear \
         ++model_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
         ++model_config.normalize=true \
+        ++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
         ++dataset_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
         ++dataset_config.dataset=audio_dataset \
         ++dataset_config.val_data_path=$inference_data_path \

examples/slam_aac/scripts/inference_audiocaps_bs.sh

@@ -6,6 +6,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 
@@ -31,6 +33,7 @@ python $code_dir/inference_aac_batch.py \
     ++model_config.encoder_projector=linear \
     ++model_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
     ++model_config.normalize=true \
+    ++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
     ++dataset_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
     ++dataset_config.dataset=audio_dataset \
     ++dataset_config.val_data_path=$inference_data_path \

examples/slam_aac/scripts/inference_clotho_CLAP_Refine.sh

@@ -6,6 +6,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 clap_dir=/data/xiquan.li/models/clap
@@ -41,6 +43,7 @@ for num_beams in "${beam_range[@]}"; do
         ++model_config.encoder_projector=linear \
         ++model_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
         ++model_config.normalize=true \
+        ++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
         ++dataset_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
         ++dataset_config.dataset=audio_dataset \
         ++dataset_config.val_data_path=$inference_data_path \

examples/slam_aac/scripts/inference_clotho_bs.sh

@@ -6,6 +6,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 
@@ -31,6 +33,7 @@ python $code_dir/inference_aac_batch.py \
     ++model_config.encoder_projector=linear \
     ++model_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
     ++model_config.normalize=true \
+    ++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
     ++dataset_config.encoder_projector_ds_rate=$encoder_projector_ds_rate \
     ++dataset_config.dataset=audio_dataset \
     ++dataset_config.val_data_path=$inference_data_path \

examples/slam_aac/scripts/pretrain.sh

@@ -9,6 +9,8 @@ run_dir=/data/wenxi.chen/SLAM-LLM
 cd $run_dir
 code_dir=examples/slam_aac
 
+encoder_fairseq_dir=/fairseq/EAT            # path to the fairseq directory of the encoder model
+
 audio_encoder_path=/data/xiquan.li/models/EAT-base_epoch30_ft.pt
 llm_path=/data/xiquan.li/models/vicuna-7b-v1.5
 
@@ -34,6 +36,7 @@ hydra.run.dir=$output_dir \
 ++model_config.encoder_path=$audio_encoder_path \
 ++model_config.encoder_dim=768 \
 ++model_config.encoder_projector=linear \
+++model_config.encoder_fairseq_dir=$encoder_fairseq_dir \
 ++dataset_config.encoder_projector_ds_rate=${encoder_projector_ds_rate} \
 ++dataset_config.dataset=audio_dataset \
 ++dataset_config.train_data_path=$train_jsonl_path \