diff --git a/apisix/plugins/ai-cache.lua b/apisix/plugins/ai-cache.lua index 5a09f0d8563b..62e69436a78a 100644 --- a/apisix/plugins/ai-cache.lua +++ b/apisix/plugins/ai-cache.lua @@ -132,6 +132,7 @@ end local function serve_hit(conf, ctx, cached, similarity) local status = "HIT" ctx.ai_cache_status = status + ctx.ai_cache_hit_layer = similarity and "semantic" or "exact" if conf.cache_headers ~= false then core.response.set_header(CACHE_STATUS_HEADER, status) local age = ngx.time() - (cached.created_at or ngx.time()) diff --git a/apisix/plugins/ai-cache/semantic.lua b/apisix/plugins/ai-cache/semantic.lua index 6cb5802834da..7c2e54da29b5 100644 --- a/apisix/plugins/ai-cache/semantic.lua +++ b/apisix/plugins/ai-cache/semantic.lua @@ -26,6 +26,7 @@ local type = type local next = next local concat = table.concat local tostring = tostring +local ngx_now = ngx.now -- Pre-require both drivers so a misconfigured provider name cannot escape -- lookup()'s fail-open boundary via a request-time require() raise. @@ -262,7 +263,9 @@ function _M.embed_query(conf, ctx, body) return nil end + local started = ngx_now() local vec, err = embed(conf, text) + ctx.ai_cache_embedding_latency = (ngx_now() - started) * 1000 if not vec then core.log.warn("ai-cache: embedding failed, fail-open as MISS: ", err) return nil diff --git a/apisix/plugins/prometheus.lua b/apisix/plugins/prometheus.lua index b1f3c58b3460..fe9e851c7b47 100644 --- a/apisix/plugins/prometheus.lua +++ b/apisix/plugins/prometheus.lua @@ -38,6 +38,7 @@ local structural_labels = { http_latency = {type = true}, bandwidth = {type = true}, llm_latency = {type = true}, + ai_cache_hits_total = {layer = true}, } diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index de3e4a8bff98..ca71a197230a 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -158,6 +158,14 @@ local metric_label_map = { "request_type", "request_llm_model", "llm_model"}, llm_completion_tokens_dist = {"route_id", "service_id", "consumer", "node", "request_type", "request_llm_model", "llm_model"}, + ai_cache_hits_total = {"layer", "route", "route_id", "service", "service_id", + "consumer", "node", "request_type", "request_llm_model", "llm_model"}, + ai_cache_misses_total = {"route", "route_id", "service", "service_id", + "consumer", "node", "request_type", "request_llm_model", "llm_model"}, + ai_cache_bypasses_total = {"route", "route_id", "service", "service_id", + "consumer", "node", "request_type", "request_llm_model", "llm_model"}, + ai_cache_embedding_latency = {"route", "route_id", "service", "service_id", + "consumer", "node", "request_type", "request_llm_model", "llm_model"}, } @@ -282,6 +290,14 @@ function _M.http_init(prometheus_enabled_in_stream) "llm_prompt_tokens_dist", "expire") local llm_completion_tokens_dist_exptime = core.table.try_read_attr(attr, "metrics", "llm_completion_tokens_dist", "expire") + local ai_cache_hits_exptime = core.table.try_read_attr(attr, "metrics", + "ai_cache_hits_total", "expire") + local ai_cache_misses_exptime = core.table.try_read_attr(attr, "metrics", + "ai_cache_misses_total", "expire") + local ai_cache_bypasses_exptime = core.table.try_read_attr(attr, "metrics", + "ai_cache_bypasses_total", "expire") + local ai_cache_embedding_latency_exptime = core.table.try_read_attr(attr, "metrics", + "ai_cache_embedding_latency", "expire") prometheus = base_prometheus.init("prometheus-metrics", metric_prefix) @@ -395,6 +411,35 @@ function _M.http_init(prometheus_enabled_in_stream) llm_completion_tokens_buckets, llm_completion_tokens_dist_exptime) + metrics.ai_cache_hits_total = prometheus:counter("ai_cache_hits_total", + "Total AI cache hits served, per cache layer", + append_tables(metric_label_map.ai_cache_hits_total, + extra_labels("ai_cache_hits_total")), + ai_cache_hits_exptime) + + metrics.ai_cache_misses_total = prometheus:counter("ai_cache_misses_total", + "Total AI cache misses", + append_tables(metric_label_map.ai_cache_misses_total, + extra_labels("ai_cache_misses_total")), + ai_cache_misses_exptime) + + metrics.ai_cache_bypasses_total = prometheus:counter("ai_cache_bypasses_total", + "Total AI cache bypassed requests", + append_tables(metric_label_map.ai_cache_bypasses_total, + extra_labels("ai_cache_bypasses_total")), + ai_cache_bypasses_exptime) + + local ai_cache_embedding_latency_buckets = DEFAULT_BUCKETS + if attr and attr.ai_cache_embedding_latency_buckets then + ai_cache_embedding_latency_buckets = attr.ai_cache_embedding_latency_buckets + end + metrics.ai_cache_embedding_latency = prometheus:histogram("ai_cache_embedding_latency", + "Latency of AI cache embedding calls in milliseconds", + append_tables(metric_label_map.ai_cache_embedding_latency, + extra_labels("ai_cache_embedding_latency")), + ai_cache_embedding_latency_buckets, + ai_cache_embedding_latency_exptime) + if prometheus_enabled_in_stream then init_stream_metrics() end @@ -425,6 +470,55 @@ function _M.stream_init() end +local AI_CACHE_STATUS_METRICS = { + HIT = "ai_cache_hits_total", + MISS = "ai_cache_misses_total", + BYPASS = "ai_cache_bypasses_total", +} + + +-- `layer` is only registered on ai_cache_hits_total, where it leads the label list +local function ai_cache_label_values(name, ctx, layer) + local vars = ctx.var + + local route_id = "" + local route_name = "" + local balancer_ip = ctx.balancer_ip or "" + local service_id = "" + local service_name = "" + local consumer_name = ctx.consumer_name or "" + + local matched_route = ctx.matched_route and ctx.matched_route.value + if matched_route then + route_id = matched_route.id + route_name = matched_route.name or "" + service_id = matched_route.service_id or "" + if service_id ~= "" then + local fetched_service = service_fetch(service_id) + service_name = fetched_service and fetched_service.value.name or "" + end + end + + local disabled_label_metric_map = get_disabled_label_metric_map() + + if layer then + return get_enabled_label_values_for_metric(name, disabled_label_metric_map, + layer, route_name, route_id, service_name, service_id, + consumer_name, balancer_ip, + vars.request_type, model_to_label(vars.request_llm_model), + model_to_label(vars.llm_model), + unpack(extra_labels(name, ctx))) + end + + return get_enabled_label_values_for_metric(name, disabled_label_metric_map, + route_name, route_id, service_name, service_id, + consumer_name, balancer_ip, + vars.request_type, model_to_label(vars.request_llm_model), + model_to_label(vars.llm_model), + unpack(extra_labels(name, ctx))) +end + + function _M.http_log(conf, ctx) local vars = ctx.var local disabled_label_metric_map = get_disabled_label_metric_map() @@ -557,6 +651,24 @@ function _M.http_log(conf, ctx) vars.request_type, request_llm_model_label, llm_model_label, unpack(extra_labels("llm_completion_tokens_dist", ctx)))) end + + local ai_cache_metric = ctx.ai_cache_status + and AI_CACHE_STATUS_METRICS[ctx.ai_cache_status] + if ai_cache_metric then + if ctx.ai_cache_status == "HIT" then + metrics[ai_cache_metric]:inc(1, + ai_cache_label_values(ai_cache_metric, ctx, + ctx.ai_cache_hit_layer or "exact")) + else + metrics[ai_cache_metric]:inc(1, + ai_cache_label_values(ai_cache_metric, ctx)) + end + end + + if ctx.ai_cache_embedding_latency then + metrics.ai_cache_embedding_latency:observe(ctx.ai_cache_embedding_latency, + ai_cache_label_values("ai_cache_embedding_latency", ctx)) + end end @@ -974,6 +1086,7 @@ function _M.dec_llm_active_connections(ctx) inc_llm_active_connections(ctx, -1) end + function _M.get_prometheus() return prometheus end diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index e31ff269e616..45a830209c8b 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -100,7 +100,7 @@ You can configure the Plugin through its [Plugin Metadata](../terminology/plugin | Name | Type | Required | Description | | --------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| disabled_labels | object | False | Per-metric map of built-in label names whose values are collapsed to an empty string `""` to reduce metric cardinality. Keyed by metric name: `http_status`, `http_latency`, `bandwidth`, `llm_latency`, `llm_prompt_tokens`, `llm_completion_tokens`, `llm_active_connections`, `llm_prompt_tokens_dist`, `llm_completion_tokens_dist`. Structural labels that define a metric's identity (`code` on `http_status`, `type` on `http_latency`, `bandwidth` and `llm_latency`) cannot be disabled. | +| disabled_labels | object | False | Per-metric map of built-in label names whose values are collapsed to an empty string `""` to reduce metric cardinality. Keyed by metric name: `http_status`, `http_latency`, `bandwidth`, `llm_latency`, `llm_prompt_tokens`, `llm_completion_tokens`, `llm_active_connections`, `llm_prompt_tokens_dist`, `llm_completion_tokens_dist`, `ai_cache_hits_total`, `ai_cache_misses_total`, `ai_cache_bypasses_total`, `ai_cache_embedding_latency`. Structural labels that define a metric's identity (`code` on `http_status`, `type` on `http_latency`, `bandwidth` and `llm_latency`, `layer` on `ai_cache_hits_total`) cannot be disabled. | Collapsing a label's value to `""` keeps the label registered in the metric schema, so existing dashboards, `absent()` alerts, and recording rules keep working — only the high-cardinality time series that differ solely by those labels are collapsed into one. This is useful in dynamic environments such as Kubernetes autoscaling, where the upstream node IP (`node` label) churns rapidly and would otherwise overflow the `prometheus-metrics` shared dict. @@ -249,6 +249,30 @@ The `type` label distinguishes the kind of latency, similar to `apisix_http_late | request_type | traditional_http / ai_chat / ai_stream | | llm_model | For non-traditional_http requests, name of the llm_model | +### Labels for the `apisix_ai_cache_*` metrics + +The [`ai-cache`](./ai-cache.md) Plugin exports four metrics: + +- `apisix_ai_cache_hits_total` counts requests served from the cache, per serving layer. +- `apisix_ai_cache_misses_total` counts requests the Plugin looked up but could not serve from the cache. +- `apisix_ai_cache_bypasses_total` counts requests that bypassed the cache lookup entirely. +- `apisix_ai_cache_embedding_latency` is a histogram of the latency, in milliseconds, of the embedding calls made by the semantic layer, measured around the embedding provider round-trip for successful and failed calls alike. + +They share the following labels: + +| Name | Description | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------- | +| layer | Only on `apisix_ai_cache_hits_total`. Cache layer that served the hit: `exact` or `semantic`. | +| route | Name of the Route that the metric corresponds to. Default to an empty string if the Route has no name or a request does not match any Route. | +| route_id | ID of the Route that the metric corresponds to. Default to an empty string if a request does not match any Route. | +| service | Name of the Service that the matched Route belongs to. Default to an empty string if the matched Route does not belong to any Service. | +| service_id | ID of the Service that the matched Route belongs to. Default to an empty string if the matched Route does not belong to any Service. | +| consumer | Name of the Consumer associated with a request. Default to an empty string if no Consumer is associated with the request. | +| node | Name of the LLM instance picked by the `ai-proxy` or `ai-proxy-multi` Plugin, such as `ai-proxy-openai`. These Plugins report the instance name instead of an upstream IP address, on cache hits and misses alike. | +| request_type | traditional_http / ai_chat / ai_stream | +| request_llm_model | Model name requested by the client. | +| llm_model | Model name reported by the LLM response. Empty on cache hits, which are served without reaching the LLM. | + ### Labels for `apisix_http_latency` The following labels are used to differentiate `apisix_http_latency` metrics. diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index f647b5d6a000..054fed1f4dda 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -100,7 +100,7 @@ plugin_attr: | 名称 | 类型 | 必选项 | 描述 | | --------------- | ------ | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| disabled_labels | object | 否 | 按指标配置的内置标签列表，列出的标签其值会被设置为空字符串 `""` 以降低指标基数。以指标名称作为键：`http_status`、`http_latency`、`bandwidth`、`llm_latency`、`llm_prompt_tokens`、`llm_completion_tokens`、`llm_active_connections`、`llm_prompt_tokens_dist`、`llm_completion_tokens_dist`。定义指标本身含义的结构性标签（`http_status` 的 `code`、`http_latency`、`bandwidth` 与 `llm_latency` 的 `type`）不可被禁用。 | +| disabled_labels | object | 否 | 按指标配置的内置标签列表，列出的标签其值会被设置为空字符串 `""` 以降低指标基数。以指标名称作为键：`http_status`、`http_latency`、`bandwidth`、`llm_latency`、`llm_prompt_tokens`、`llm_completion_tokens`、`llm_active_connections`、`llm_prompt_tokens_dist`、`llm_completion_tokens_dist`、`ai_cache_hits_total`、`ai_cache_misses_total`、`ai_cache_bypasses_total`、`ai_cache_embedding_latency`。定义指标本身含义的结构性标签（`http_status` 的 `code`、`http_latency`、`bandwidth` 与 `llm_latency` 的 `type`、`ai_cache_hits_total` 的 `layer`）不可被禁用。 | 将标签值设置为 `""` 时，标签仍保留在指标 schema 中，因此现有的仪表盘、`absent()` 告警和 recording rule 都不受影响——只是将仅因这些标签而不同的高基数时间序列合并为一条。这在 Kubernetes 弹性伸缩等动态环境中尤其有用：此时上游节点 IP（`node` 标签）频繁变化，否则会很快撑爆 `prometheus-metrics` 共享字典。 @@ -249,6 +249,30 @@ Prometheus 中有不同类型的指标。要了解它们之间的区别，请参 | request_type | traditional_http / ai_chat / ai_stream | | llm_model | 对于非传统的 http 请求，llm 模型的名称 | +### `apisix_ai_cache_*` 系列指标的标签 + +[`ai-cache`](./ai-cache.md) 插件导出以下四个指标： + +- `apisix_ai_cache_hits_total`：统计由缓存命中并直接返回的请求数，按命中的缓存层区分。 +- `apisix_ai_cache_misses_total`：统计经过插件查询但未命中缓存的请求数。 +- `apisix_ai_cache_bypasses_total`：统计完全绕过缓存查询的请求数。 +- `apisix_ai_cache_embedding_latency`：语义层发起的 embedding 调用延迟（毫秒）的直方图，围绕 embedding 服务的完整往返计时，成功与失败的调用均会记录。 + +它们共享以下标签： + +| 名称 | 描述 | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------- | +| layer | 仅存在于 `apisix_ai_cache_hits_total`。命中的缓存层：`exact` 或 `semantic`。 | +| route | 指标对应的路由名称。如果路由未配置名称或请求不匹配任何路由，则默认为空字符串。 | +| route_id | 指标对应的路由 ID。如果请求不匹配任何路由，则默认为空字符串。 | +| service | 匹配路由所属的服务名称。如果匹配的路由不属于任何服务，则默认为空字符串。 | +| service_id | 匹配路由所属的服务 ID。如果匹配的路由不属于任何服务，则默认为空字符串。 | +| consumer | 与请求关联的消费者名称。如果请求没有与之关联的消费者，则默认为空字符串。 | +| node | `ai-proxy` 或 `ai-proxy-multi` 插件选中的 LLM 实例名称，例如 `ai-proxy-openai`。这些插件上报的是实例名称而非上游 IP 地址，缓存命中与未命中时均是如此。 | +| request_type | traditional_http / ai_chat / ai_stream | +| request_llm_model | 客户端请求的模型名称。 | +| llm_model | LLM 响应中报告的模型名称。缓存命中的请求不会到达 LLM，此标签为空字符串。 | + ### `apisix_http_latency` 的标签 以下标签用于区分 `apisix_http_latency` 指标。 diff --git a/t/plugin/prometheus-ai-cache.t b/t/plugin/prometheus-ai-cache.t new file mode 100644 index 000000000000..31e07f2eaec9 --- /dev/null +++ b/t/plugin/prometheus-ai-cache.t @@ -0,0 +1,270 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +BEGIN { + if ($ENV{TEST_NGINX_CHECK_LEAK}) { + $SkipReason = "unavailable for the hup tests"; + + } else { + $ENV{TEST_NGINX_USE_HUP} = 1; + undef $ENV{TEST_NGINX_USE_STAP}; + } +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $extra_yaml_config = <<_EOC_; +plugin_attr: + prometheus: + refresh_interval: 0.1 +plugins: + - ai-proxy + - ai-cache + - prometheus + - public-api +_EOC_ + $block->set_value("extra_yaml_config", $extra_yaml_config); + + my $http_config = <<_EOC_; + server { + listen 6724; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { require("lib.ai_cache_mock").embeddings() } + } + } +_EOC_ + $block->set_value("http_config", $http_config); +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create a route with ai-proxy, ai-cache and the metrics public-api route +--- config + location /t { + content_by_lua_block { + require("lib.test_redis").flush_port("127.0.0.1", 6379) + + local data = { + { + url = "/apisix/admin/routes/1", + data = [[{ + "uri": "/chat", + "name": "ai-cache-route", + "plugins": { + "prometheus": {}, + "ai-proxy": { + "provider": "openai", + "auth": { "header": { "Authorization": "Bearer test-key" } }, + "options": { "model": "gpt-4o" }, + "override": { "endpoint": "http://127.0.0.1:1980" } + }, + "ai-cache": { + "redis_host": "127.0.0.1", + "redis_port": 6379, + "layers": ["exact", "semantic"], + "bypass_on": [{"header": "X-No-Cache", "equals": "1"}], + "semantic": { + "similarity_threshold": 0.9, + "embedding": { + "openai": { + "endpoint": "http://127.0.0.1:6724/v1/embeddings", + "model": "text-embedding-3-small", + "api_key": "test-key" + } + }, + "vector_search": { "redis": {} } + } + } + } + }]], + }, + { + url = "/apisix/admin/routes/metrics", + data = [[{ + "plugins": { + "public-api": {} + }, + "uri": "/apisix/prometheus/metrics" + }]] + }, + } + + local t = require("lib.test_admin").test + + for _, d in ipairs(data) do + local code, body = t(d.url, ngx.HTTP_PUT, d.data) + if code >= 300 then ngx.status = code end + ngx.say(body) + end + } + } +--- response_body eval +"passed\n" x 2 + + + +=== TEST 2: send a chat request (cache MISS) +--- request +POST /chat +{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of France?"}]} +--- more_headers +X-AI-Fixture: openai/chat-basic.json +--- response_headers +X-AI-Cache-Status: MISS +--- error_code: 200 +--- wait: 0.5 + + + +=== TEST 3: assert ai_cache_misses_total metric +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_misses_total\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model="gpt-4o"\} 1/ + + + +=== TEST 4: send the same chat request (exact-layer HIT) +--- request +POST /chat +{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of France?"}]} +--- response_headers +X-AI-Cache-Status: HIT +! X-AI-Cache-Similarity +--- error_code: 200 +--- wait: 0.3 + + + +=== TEST 5: assert ai_cache_hits_total metric with layer=exact and empty llm_model +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_hits_total\{layer="exact",route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model=""\} 1/ + + + +=== TEST 6: ai_cache_embedding_latency is not recorded for the exact hit +--- request +GET /apisix/prometheus/metrics +--- response_body_unlike eval +qr/apisix_ai_cache_embedding_latency_count\{[^}]*llm_model=""\}/ + + + +=== TEST 7: send a paraphrased chat request (semantic-layer HIT, cos 0.922 >= 0.9) +--- request +POST /chat +{"model":"gpt-4o","messages":[{"role":"user","content":"What's the capital city of France?"}]} +--- response_headers_like +X-AI-Cache-Status: HIT +X-AI-Cache-Similarity: 0\.92\d\d +--- error_code: 200 +--- wait: 0.3 + + + +=== TEST 8: assert ai_cache_hits_total metric with layer=semantic +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_hits_total\{layer="semantic",route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model=""\} 1/ + + + +=== TEST 9: assert ai_cache_embedding_latency_bucket metric for the miss lookup +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_embedding_latency_bucket\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model="gpt-4o",le="\d+"\} 1/ + + + +=== TEST 10: assert ai_cache_embedding_latency_count metric for the miss lookup +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_embedding_latency_count\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model="gpt-4o"\} 1/ + + + +=== TEST 11: assert ai_cache_embedding_latency_sum metric for the miss lookup +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_embedding_latency_sum\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model="gpt-4o"\} \d+/ + + + +=== TEST 12: assert ai_cache_embedding_latency_count metric for the semantic-hit lookup +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_embedding_latency_count\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model=""\} 1/ + + + +=== TEST 13: send a chat request with the bypass header (BYPASS) +--- request +POST /chat +{"model":"gpt-4o","messages":[{"role":"user","content":"What is the capital of France?"}]} +--- more_headers +X-No-Cache: 1 +X-AI-Fixture: openai/chat-basic.json +--- response_headers +X-AI-Cache-Status: BYPASS +--- error_code: 200 +--- wait: 0.3 + + + +=== TEST 14: assert ai_cache_bypasses_total metric +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_ai_cache_bypasses_total\{route="ai-cache-route",route_id="1",.*node="ai-proxy-openai",request_type="ai_chat",request_llm_model="gpt-4o",llm_model="gpt-4o"\} 1/ + + + +=== TEST 15: reject disabling the structural `layer` label on ai_cache_hits_total +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/prometheus', + ngx.HTTP_PUT, + [[{"disabled_labels": {"ai_cache_hits_total": ["layer"]}}]]) + ngx.say(body) + } + } +--- response_body eval +qr/failed to validate item 1/