ci: Create weekly dev branch (NVIDIA#2223)

Signed-off-by: oliver könig <[email protected]>
Signed-off-by: GitHub Actions <github-actions[bot]@users.noreply.github.com>
Co-authored-by: GitHub Actions <github-actions[bot]@users.noreply.github.com>

Author: ko3n1g

.gitlab/stages/00.pre.yml

@@ -68,6 +68,7 @@ pre:create_ci_branches_dev:
       - branch: ci-dev-rebuild-mcore-nemo-image
       - branch: ci-dev-mr
       - branch: ci-dev-nightly
+      - branch: ci-dev-weekly
       - branch: ci-dev-upgrade-dependencies
   tags:
     - arch/amd64

.gitlab/stages/04.functional-tests.yml

@@ -53,7 +53,7 @@ functional:configure:
     - |
       RECORD_CHECKPOINTS=$([[ "$CI_MERGE_REQUEST_LABELS" == *"Record checkpoints"* || "$FUNCTIONAL_TEST_RECORD_CHECKPOINTS" == "yes" ]] && echo "true" || echo "false")
     - |
-      if [[ "$FUNCTIONAL_TEST_SCOPE" == "release" || "$FUNCTIONAL_TEST_SCOPE" == "pre-release" ]]; then
+      if [[ "$FUNCTIONAL_TEST_SCOPE" == "release" || "$FUNCTIONAL_TEST_SCOPE" == "weekly" ]]; then
         FUNCTIONAL_TEST_NAME=$(eval echo $FUNCTIONAL_TEST_NAME)
         RELEASE_ARGS=(
           "--run-name"

tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml

@@ -23,7 +23,7 @@ MODEL_ARGS:
   --micro-batch-size: 4
   --rampup-batch-size: "[384 384 97656250]"
   --global-batch-size: 1152
-  --train-samples: 4882812
+  --train-samples: 19531250
   --manual-gc: true
   # Transformer Engine args
   --transformer-impl: transformer_engine
@@ -68,9 +68,10 @@ MODEL_ARGS:
   --eval-iters: 32
   --eval-interval: 2000
   # Add checkpointing args
-  --load: ${CHECKPOINT_LOAD_PATH}
   --save: ${CHECKPOINT_SAVE_PATH}
-  --save-interval: 1000
+  --load: ${CHECKPOINT_LOAD_PATH}
+  --save-interval: 5000
+  --save-retain-interval: 10000
   # Add initialization args
   --init-method-std: 0.0134
   # Add logging args
@@ -86,7 +87,7 @@ MODEL_ARGS:
   --wandb-exp-name: ${WANDB_EXPERIMENT}
   # Add mixed precision args
   --bf16: true
-  --exit-interval: 13000
+  --exit-interval: 10200
   --wandb-save-dir: ${WANDB_SAVE_PATH}
 METRICS:
   - "iteration-time"

tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml

@@ -41,7 +41,7 @@ MODEL_ARGS:
   --pipeline-model-parallel-size: 2
   --sequence-parallel: true
   --tp-comm-overlap: true
-  --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_b200_1N8G_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
+  --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml
   --deterministic-mode: true
   --no-gradient-accumulation-fusion: true
   --fp8-format: hybrid