chore: cortex.cpp gpu activation could cause a race condition (#4825)

Author: louis-jan

extensions/conversational-extension/src/index.ts

@@ -22,22 +22,26 @@ type MessageList = {
 export default class CortexConversationalExtension extends ConversationalExtension {
   queue = new PQueue({ concurrency: 1 })
 
+  api?: KyInstance
   /**
-   * Extended API instance for making requests to the Cortex API.
+   * Get the API instance
    * @returns
    */
-  api: KyInstance
+  async apiInstance(): Promise<KyInstance> {
+    if(this.api) return this.api
+    const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
+    this.api = ky.extend({
+      prefixUrl: API_URL,
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    })
+    return this.api
+  }
   /**
    * Called when the extension is loaded.
    */
   async onLoad() {
-    const apiKey = await window.core?.api.appToken() ?? 'cortex.cpp'
-    this.api = ky.extend({
-        prefixUrl: API_URL,
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-        },
-      })
     this.queue.add(() => this.healthz())
   }
 
@@ -51,10 +55,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async listThreads(): Promise<Thread[]> {
     return this.queue.add(() =>
-      this.api
-        .get('v1/threads?limit=-1')
-        .json<ThreadList>()
-        .then((e) => e.data)
+      this.apiInstance().then((api) =>
+        api
+          .get('v1/threads?limit=-1')
+          .json<ThreadList>()
+          .then((e) => e.data)
+      )
     ) as Promise<Thread[]>
   }
 
@@ -64,7 +70,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async createThread(thread: Thread): Promise<Thread> {
     return this.queue.add(() =>
-      this.api.post('v1/threads', { json: thread }).json<Thread>()
+      this.apiInstance().then((api) =>
+        api.post('v1/threads', { json: thread }).json<Thread>()
+      )
     ) as Promise<Thread>
   }
 
@@ -75,7 +83,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
   async modifyThread(thread: Thread): Promise<void> {
     return this.queue
       .add(() =>
-        this.api.patch(`v1/threads/${thread.id}`, { json: thread })
+        this.apiInstance().then((api) =>
+          api.patch(`v1/threads/${thread.id}`, { json: thread })
+        )
       )
       .then()
   }
@@ -86,7 +96,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async deleteThread(threadId: string): Promise<void> {
     return this.queue
-      .add(() => this.api.delete(`v1/threads/${threadId}`))
+      .add(() =>
+        this.apiInstance().then((api) => api.delete(`v1/threads/${threadId}`))
+      )
       .then()
   }
 
@@ -97,11 +109,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async createMessage(message: ThreadMessage): Promise<ThreadMessage> {
     return this.queue.add(() =>
-      this.api
-        .post(`v1/threads/${message.thread_id}/messages`, {
-          json: message,
-        })
-        .json<ThreadMessage>()
+      this.apiInstance().then((api) =>
+        api
+          .post(`v1/threads/${message.thread_id}/messages`, {
+            json: message,
+          })
+          .json<ThreadMessage>()
+      )
     ) as Promise<ThreadMessage>
   }
 
@@ -112,14 +126,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> {
     return this.queue.add(() =>
-      this.api
-        .patch(
-          `v1/threads/${message.thread_id}/messages/${message.id}`,
-          {
+      this.apiInstance().then((api) =>
+        api
+          .patch(`v1/threads/${message.thread_id}/messages/${message.id}`, {
             json: message,
-          }
-        )
-        .json<ThreadMessage>()
+          })
+          .json<ThreadMessage>()
+      )
     ) as Promise<ThreadMessage>
   }
 
@@ -132,7 +145,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
   async deleteMessage(threadId: string, messageId: string): Promise<void> {
     return this.queue
       .add(() =>
-        this.api.delete(`v1/threads/${threadId}/messages/${messageId}`)
+        this.apiInstance().then((api) =>
+          api.delete(`v1/threads/${threadId}/messages/${messageId}`)
+        )
       )
       .then()
   }
@@ -144,10 +159,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async listMessages(threadId: string): Promise<ThreadMessage[]> {
     return this.queue.add(() =>
-      this.api
-        .get(`v1/threads/${threadId}/messages?order=asc&limit=-1`)
-        .json<MessageList>()
-        .then((e) => e.data)
+      this.apiInstance().then((api) =>
+        api
+          .get(`v1/threads/${threadId}/messages?order=asc&limit=-1`)
+          .json<MessageList>()
+          .then((e) => e.data)
+      )
     ) as Promise<ThreadMessage[]>
   }
 
@@ -159,9 +176,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    */
   async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> {
     return this.queue.add(() =>
-      this.api
-        .get(`v1/assistants/${threadId}?limit=-1`)
-        .json<ThreadAssistantInfo>()
+      this.apiInstance().then((api) =>
+        api
+          .get(`v1/assistants/${threadId}?limit=-1`)
+          .json<ThreadAssistantInfo>()
+      )
     ) as Promise<ThreadAssistantInfo>
   }
   /**
@@ -175,9 +194,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
     assistant: ThreadAssistantInfo
   ): Promise<ThreadAssistantInfo> {
     return this.queue.add(() =>
-      this.api
-        .post(`v1/assistants/${threadId}`, { json: assistant })
-        .json<ThreadAssistantInfo>()
+      this.apiInstance().then((api) =>
+        api
+          .post(`v1/assistants/${threadId}`, { json: assistant })
+          .json<ThreadAssistantInfo>()
+      )
     ) as Promise<ThreadAssistantInfo>
   }
 
@@ -192,9 +213,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
     assistant: ThreadAssistantInfo
   ): Promise<ThreadAssistantInfo> {
     return this.queue.add(() =>
-      this.api
-        .patch(`v1/assistants/${threadId}`, { json: assistant })
-        .json<ThreadAssistantInfo>()
+      this.apiInstance().then((api) =>
+        api
+          .patch(`v1/assistants/${threadId}`, { json: assistant })
+          .json<ThreadAssistantInfo>()
+      )
     ) as Promise<ThreadAssistantInfo>
   }
 
@@ -203,10 +226,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
    * @returns
    */
   async healthz(): Promise<void> {
-    return this.api
-      .get('healthz', {
-        retry: { limit: 20, delay: () => 500, methods: ['get'] },
-      })
+    return this.apiInstance()
+      .then((api) =>
+        api.get('healthz', {
+          retry: { limit: 20, delay: () => 500, methods: ['get'] },
+        })
+      )
       .then(() => {})
   }
 }