diff --git a/calibration/step-1-prepare-calibration-dataset.py b/calibration/step-1-prepare-calibration-dataset.py
index 01bbdfe5a..dd2b464a7 100755
--- a/calibration/step-1-prepare-calibration-dataset.py
+++ b/calibration/step-1-prepare-calibration-dataset.py
@@ -39,6 +39,10 @@ def main(args):
             model_max_length=args.max_model_length,
             padding_side="left",
             use_fast=False,)
+
+        # Llama4 models in Text Only mode fail with False if use_fast=False
+        if isinstance(tokenizer, bool):
+            raise ValueError("Tokenizer is a boolean")
     except:
         tokenizer = transformers.AutoTokenizer.from_pretrained(
             args.model,