add apriel

Author: foldl

README.md

@@ -13,6 +13,7 @@ pure C++ implementation based on [@ggerganov](https://github.com/ggerganov)'s [g
 
 **What's New:**
 
+* 2025-04-18: Apriel
 * 2025-04-15: GLM-4-0414
 * 2025-04-10: LlaMA4 (Language model)
 * 2025-03-27: Ling (Bailing)

convert.py

@@ -1307,12 +1307,13 @@ def dump_config(f, config, ggml_type):
         f.write(struct.pack("i" * len(config_values), *config_values))
         config_values = [
             config.rope_theta,
+            config.head_dim,
             config.rope_scaling['original_max_position_embeddings'],
             config.rope_scaling['beta_fast'],
             config.rope_scaling['beta_slow'],
             config.rope_scaling['factor'],
         ]
-        f.write(struct.pack("<fifff", *config_values))
+        f.write(struct.pack("<fiifff", *config_values))
 
     @staticmethod
     def get_weight_names(config):

docs/models.md

@@ -5,6 +5,9 @@
 * Adept Persimmon (`PersimmonForCausalLM`)
     * [x] [Chat-8B](https://huggingface.co/adept/persimmon-8b-chat)
 
+* Apriel (`AprielForCausalLM`)
+    * [x] [Instruct-5B](https://huggingface.co/ServiceNow-AI/Apriel-5B-Instruct/tree/a9a4831718a2fad437f25ace0d0259953fcaaa26)
+
 * Aquila (`AquilaForCausalLM`)
     * [x] [Chat2-7B](https://huggingface.co/BAAI/AquilaChat2-7B/tree/9905960de19ea9e573c0dc3fbdf54d4ddcc610d3), [Chat2-34B](https://huggingface.co/BAAI/AquilaChat2-34B/commit/5c7990b198c94b63dfbfa022462b9cf672dbcfa0), [Chat2-7B-16K](https://huggingface.co/BAAI/AquilaChat2-7B-16K/commit/fb46d48479d05086ccf6952f19018322fcbb54cd), [Chat2-34B-16K](https://huggingface.co/BAAI/AquilaChat2-34B-16K/tree/9f19774f3e7afad2fc3d51fe308eac5a2d88c8b1)
 

models/apriel.cpp

@@ -0,0 +1,95 @@
+struct Config : public llama::v3::Config
+{
+    int head_dim;
+    int rope_scaling_original_max_position_embeddings;
+    float rope_scaling_beta_fast;
+    float rope_scaling_beta_slow;
+    float rope_scaling_factor;
+};
+
+class ChatHistoryEncoder : public BaseHistoryEncoder
+{
+public:
+    void append_sys_prompt(std::vector<int> &ids) const override
+    {
+        std::ostringstream oss;
+        ids.push_back(tokenizer->bos_token_id);
+        oss << "<|system|>\n" << tokenizer->get_system_prompt() << "\n<|end|>\n";
+        tokenizer->encode(oss.str(), ids);
+    }
+    void append_ai(int round_idx, const std::string &ai, std::vector<int> &ids) const override
+    {
+        append_ai_opening(round_idx, ids);
+        tokenizer->encode(ai, ids);
+        tokenizer->encode("\n<|end|>\n", ids);
+    }
+
+    void append_user(int round_idx, const std::string &user, std::vector<int> &ids) const override
+    {
+        append_user_opening(round_idx, ids);
+        tokenizer->encode(user, ids);
+        tokenizer->encode("\n<|end|>\n", ids);
+    }
+
+    void append_ai_opening(int round_idx, std::vector<int> &ids) const override
+    {
+        tokenizer->encode("<|assistant|>\n", ids);
+    }
+
+    void append_user_opening(int round_idx, std::vector<int> &ids) const override
+    {
+        tokenizer->encode("<|user|>\n", ids);
+    }
+};
+
+static ChatHistoryEncoder _chat_encoder;
+
+class Tokenizer : public BaseTokenizer
+{
+public:
+    Tokenizer(const BaseConfig &config)
+        : Tokenizer(config, &_chat_encoder)
+    {}
+
+    Tokenizer(const BaseConfig &config, BaseHistoryEncoder *encoder)
+        : BaseTokenizer::BaseTokenizer(config, encoder)
+    {
+        sys_prompt = "You are a helpful AI assistant that provides accurate and concise information.";
+    }
+
+    size_t load(tokenizer::DataReader *buffer, int n_vocab) override
+    {
+        tp = new tokenizer::BPEProcessor2(
+            {
+                "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+            }
+        );
+        size_t size = tp->Load(buffer, n_vocab);
+
+        return size;
+    }
+};
+
+class ConditionalGeneration : public llama::v2::GenericConditionalGeneration<LlamaBlock>
+{
+public:
+    ConditionalGeneration() = default;
+    ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config, ModelType type = ModelType::MODEL_TYPE_APRIEL)
+        : llama::v2::GenericConditionalGeneration<LlamaBlock>(config, runtime_config, type, config.num_key_value_heads, config.head_dim, config.max_length, 12, false)
+    {
+        auto transformer = Base::get_typed_transformer<ModelClass2>();
+        for (int i = 0; i < config.num_hidden_layers; i++)
+        {
+            auto &attention = transformer->layers[i].attention;
+            attention.freq_base = config.rope_theta;
+
+            attention.n_original_ctx = config.rope_scaling_original_max_position_embeddings;
+            attention.beta_fast = config.rope_scaling_beta_fast;
+            attention.beta_slow = config.rope_scaling_beta_slow;
+
+            attention.freq_scale  = 1 / config.rope_scaling_factor;
+            attention.attn_factor = 1.0f;
+            attention.ext_factor  = 1.0f;
+        }
+    }
+};

scripts/models.json

@@ -2463,5 +2463,21 @@
                 }
             }
         }
+    },
+    "apriel": {
+        "brief": "Apriel is a family of models built for versatility, offering high throughput and efficiency across a wide range of tasks.",
+        "default": "5b",
+        "license": "MIT",
+        "variants": {
+            "5b": {
+                "default": "q8",
+                "quantized": {
+                    "q8": {
+                        "size": 5140774832,
+                        "url": "chatllm_quantized_apriel/apriel-5b.bin"
+                    }
+                }
+            }
+        }
     }
 }

src/layers.h

@@ -1565,7 +1565,6 @@ namespace chatllm
               beta_fast(0.0f),
               beta_slow(0.0f),
               rope_dim(head_dim),
-              n_ctx(0),
               n_original_ctx(0),
               mrope_sections(nullptr),
               use_rope(true),
@@ -1586,7 +1585,6 @@ namespace chatllm
               beta_fast(0.0f),
               beta_slow(0.0f),
               rope_dim(rope_dim),
-              n_ctx(0),
               n_original_ctx(0),
               mrope_sections(nullptr),
               use_rope(true),
@@ -1958,6 +1956,10 @@ namespace chatllm
         LlamaBlock(InitContext *ctx, int hidden_size, int num_attention_heads, int intermediate_size, int num_kv_heads, int max_length)
             : LMBlock1(ctx, hidden_size, num_attention_heads, intermediate_size, num_kv_heads, max_length)
         {}
+
+        LlamaBlock(InitContext *ctx, int hidden_size, int num_attention_heads, int intermediate_size, int num_kv_heads, int head_dim, int max_length)
+            : LMBlock1(ctx, hidden_size, num_attention_heads, intermediate_size, num_kv_heads, head_dim, max_length)
+        {}
     };
 
     class Llama31SelfAttention : public RoPESelfAttention<BaseAttention>

src/models.cpp

@@ -343,6 +343,8 @@ namespace chatllm
 
         MODEL_TYPE_SOLARPRO         = 0x2300,
 
+        MODEL_TYPE_APRIEL           = 0x2400,
+
         MODEL_TYPE_BCE_Embedding = 0x10000100,
         MODEL_TYPE_BCE_ReRanker  = 0x10000101,
         MODEL_TYPE_BGE_M3        = 0x10000102,
@@ -2018,6 +2020,11 @@ namespace chatllm
         #include "../models/kimi.cpp"
     }
 
+    namespace apriel
+    {
+        #include "../models/apriel.cpp"
+    }
+
     template <class Config>
     void load_config(ModelLoader &loader, Config &config, const ModelObject::extra_args &args)
     {
@@ -2407,6 +2414,8 @@ namespace chatllm
                                                                 \
         CASE(KIMI_VL,               kimi::vl, 1)                \
                                                                 \
+        CASE(APRIEL,                apriel, 1)                  \
+                                                                \
         CASE(BCE_Embedding,         bce::embedding, 1)          \
         CASE(BCE_ReRanker,          bce::ranker, 1)             \
         CASE(BGE_M3,                bge::embedding, 1)          \