-
Notifications
You must be signed in to change notification settings - Fork 104
Adds custom inference service API docs #4852
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
65cb119
ac77396
389ce57
a9a560e
3c4eb3f
3d90b42
bc66328
1b3fe33
fab50c4
e185149
859713d
8051083
61c6a98
e0963ae
bd8ec2b
d568234
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
{ | ||
"inference.put_custom": { | ||
"documentation": { | ||
"url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put-custom", | ||
"description": "Configure a custom inference endpoint" | ||
}, | ||
"stability": "stable", | ||
"visibility": "public", | ||
"headers": { | ||
"accept": ["application/json"], | ||
"content_type": ["application/json"] | ||
}, | ||
"url": { | ||
"paths": [ | ||
{ | ||
"path": "/_inference/{task_type}/{custom_inference_id}", | ||
"methods": ["PUT"], | ||
"parts": { | ||
"task_type": { | ||
"type": "string", | ||
"description": "The task type" | ||
}, | ||
"custom_inference_id": { | ||
"type": "string", | ||
"description": "The inference Id" | ||
} | ||
} | ||
} | ||
] | ||
}, | ||
"body": { | ||
"description": "The inference endpoint's task and service settings" | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -758,6 +758,136 @@ export class CohereTaskSettings { | |
truncate?: CohereTruncateType | ||
} | ||
|
||
export class CustomServiceSettings { | ||
/** | ||
* Specifies the HTTPS header parameters – such as `Authentication` or `Contet-Type` – that are required to access the custom service. | ||
* For example: | ||
* ``` | ||
* "headers":{ | ||
* "Authorization": "Bearer ${api_key}", | ||
* "Content-Type": "application/json;charset=utf-8" | ||
* } | ||
* ``` | ||
*/ | ||
headers?: UserDefinedValue | ||
/** | ||
* The request configuration object. | ||
*/ | ||
request: CustomRequestParams | ||
/** | ||
* The response configuration object. | ||
*/ | ||
response: CustomResponseParams | ||
/** | ||
* Specifies secret parameters, like `api_key` or `api_token`, that are required to access the custom service. | ||
* For example: | ||
* ``` | ||
* "secret_parameters":{ | ||
* "api_key":"<api_key>" | ||
* } | ||
* ``` | ||
*/ | ||
secret_parameters: UserDefinedValue | ||
/** | ||
* The URL endpoint to use for the requests. | ||
*/ | ||
url?: string | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We also parse an optional
If the subsequent inference requests come from a search context we'll use the The keys we allow in
This is particularly useful for integrations like Cohere that allow an input type field in their API: https://docs.cohere.com/reference/embed#request.body.input_type |
||
|
||
export class CustomRequestParams { | ||
/** | ||
* The body structure of the request. It requires passing in the string-escaped result of the JSON format HTTP request body. | ||
* For example: | ||
* ``` | ||
* "request":{ | ||
* "content":"{\"input\":${input}}" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We flattened this so it's |
||
* } | ||
* ``` | ||
* > info | ||
* > The content string needs to be a single line except using the Kibana console. | ||
*/ | ||
content: string | ||
} | ||
|
||
export class CustomResponseParams { | ||
/** | ||
* Specifies the path to the error message in the response from the custom service. | ||
* For example: | ||
* ``` | ||
* "response": { | ||
* "error_parser": { | ||
* "path": "$.error.message" | ||
* } | ||
* } | ||
* ``` | ||
*/ | ||
error_parser: UserDefinedValue | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can remove this field, we simplified the error handling logic and removed this. |
||
/** | ||
* Specifies the JSON parser that is used to parse the response from the custom service. | ||
* Different task types require different json_parser parameters. | ||
* For example: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jonathan-buttner Do you think we should specify a JsonParser class for each task type, or is this list sufficient? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm I think it might be better if we give an example of the response structure for each task type and explain how to create the parser from that. We should also say that the format is a less featured version of JSONPath: https://en.wikipedia.org/wiki/JSONPath Here are some examples: Text EmbeddingsFor a response that looks like:
We'd need this definition:
RerankFor a response that looks like:
We'd need this definition:
CompletionFor a response that looks like:
We'd need this definition:
Sparse embeddingFor a response that looks like:
We'd need this definition:
If the |
||
* ``` | ||
* # text_embedding | ||
* "response":{ | ||
* "json_parser":{ | ||
* "text_embeddings":"$.result.embeddings[*].embedding" | ||
* } | ||
* } | ||
* | ||
* # sparse_embedding | ||
* "response":{ | ||
* "json_parser":{ | ||
* "token_path":"$.result[*].embeddings[*].token", | ||
* "weight_path":"$.result[*].embeddings[*].weight" | ||
* } | ||
* } | ||
* | ||
* # rerank | ||
* "response":{ | ||
* "json_parser":{ | ||
* "reranked_index":"$.result.scores[*].index", // optional | ||
* "relevance_score":"$.result.scores[*].score", | ||
* "document_text":"xxx" // optional | ||
* } | ||
* } | ||
* | ||
* # completion | ||
* "response":{ | ||
* "json_parser":{ | ||
* "completion_result":"$.result.text" | ||
* } | ||
* } | ||
*/ | ||
json_parser: UserDefinedValue | ||
} | ||
|
||
export enum CustomTaskType { | ||
text_embedding, | ||
sparse_embedding, | ||
rerank, | ||
completion | ||
} | ||
|
||
export enum CustomServiceType { | ||
custom | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jonathan-buttner Should the ServiceType be There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah it must be |
||
} | ||
|
||
export class CustomTaskSettings { | ||
/** | ||
* Specifies parameters that are required to run the custom service. The parameters depend on the model your custom service uses. | ||
* For example: | ||
* ``` | ||
* "task_settings":{ | ||
* "parameters":{ | ||
* "input_type":"query", | ||
* "return_token":true | ||
* } | ||
* } | ||
* ``` | ||
*/ | ||
parameters?: UserDefinedValue | ||
} | ||
|
||
export class EisServiceSettings { | ||
/** | ||
* The name of the model to use for the inference task. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
/* | ||
* Licensed to Elasticsearch B.V. under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch B.V. licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
import { RequestBase } from '@_types/Base' | ||
import { Id } from '@_types/common' | ||
import { | ||
CustomServiceSettings, | ||
CustomServiceType, | ||
CustomTaskSettings, | ||
CustomTaskType | ||
} from '@inference/_types/CommonTypes' | ||
import { InferenceChunkingSettings } from '@inference/_types/Services' | ||
|
||
/** | ||
* Create a custom inference endpoint. | ||
* | ||
* You can create an inference endpoint to perform an inference task with a custom model that supports the HTTP format. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jonathan-buttner Please suggest an alternative description if you think this is not sufficient. I tried to come up with something that is meaningful to me based on my limited knowledge. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm maybe something like this: The custom service gives more control over how to interact with external inference services that aren't explicitly supported through dedicated integrations. The custom service gives users the ability to define the headers, url, query parameters, request body, and secrets. |
||
* @rest_spec_name inference.put_custom | ||
* @availability stack since=8.13.0 stability=stable visibility=public | ||
* @availability serverless stability=stable visibility=public | ||
* @cluster_privileges manage_inference | ||
* @doc_id inference-api-put-custom | ||
*/ | ||
export interface Request extends RequestBase { | ||
urls: [ | ||
{ | ||
path: '/_inference/{task_type}/{custom_inference_id}' | ||
methods: ['PUT'] | ||
} | ||
] | ||
path_parts: { | ||
/** | ||
* The type of the inference task that the model will perform. | ||
*/ | ||
task_type: CustomTaskType | ||
/** | ||
* The unique identifier of the inference endpoint. | ||
*/ | ||
custom_inference_id: Id | ||
} | ||
body: { | ||
/** | ||
* The chunking configuration object. | ||
* @ext_doc_id inference-chunking | ||
*/ | ||
chunking_settings?: InferenceChunkingSettings | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are chunking settings relevant for this service? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep! |
||
/** | ||
* The type of service supported for the specified task type. In this case, `custom`. | ||
*/ | ||
service: CustomServiceType | ||
/** | ||
* Settings used to install the inference model. | ||
* These settings are specific to the `custom` service. | ||
*/ | ||
service_settings: CustomServiceSettings | ||
/** | ||
* Settings to configure the inference task. | ||
* These settings are specific to the task type you specified. | ||
*/ | ||
task_settings?: CustomTaskSettings | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
/* | ||
* Licensed to Elasticsearch B.V. under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch B.V. licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
import { InferenceEndpointInfoCustom } from '@inference/_types/Services' | ||
|
||
export class Response { | ||
/** @codegen_name endpoint_info */ | ||
body: InferenceEndpointInfoCustom | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
summary: Custom text embedding task (OpenAI) | ||
description: Run `PUT _inference/text_embedding/custom-embeddings` to create an inference endpoint that performs a text embedding task. | ||
method_request: 'PUT _inference/text_embedding/custom-embeddings' | ||
# type: "request" | ||
value: |- | ||
{ | ||
"service": "custom", | ||
"service_settings": { | ||
"secret_parameters": { | ||
"api_key": "<api key>" | ||
}, | ||
"url": "https://api.openai.com/v1/embeddings", | ||
"headers": { | ||
"Authorization": "Bearer ${api_key}", | ||
"Content-Type": "application/json;charset=utf-8" | ||
}, | ||
"request": "{\"input\": ${input}, \"model\": \"text-embedding-3-small\"}", | ||
"response": { | ||
"json_parser": { | ||
"text_embeddings": "$.data[*].embedding[*]" | ||
} | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We also support query parameters:
It's a list of tuples (the inner array must be 2 items).
This would be invalid:
This is valid: