Rag/errata

assets/css/app.scss

@@ -78,3 +78,35 @@
 );
 
 @use '~bitstyles/scss/bitstyles/utilities/z-index';
+
+.loader {
+  width: 1rem;
+  height: 1rem;
+  border: 5px solid #9485ff;
+  border-bottom-color: transparent;
+  border-radius: 50%;
+  display: inline-block;
+  box-sizing: border-box;
+  animation: rotation 1s linear infinite;
+  }
+
+  @keyframes rotation {
+  0% {
+      transform: rotate(0deg);
+  }
+  100% {
+      transform: rotate(360deg);
+  }
+} 
+
+.a-card ol {
+  padding-inline-start: var(--bs-size-l2);
+}
+
+.a-card ul {
+  padding-inline-start: var(--bs-size-l2);
+}
+
+.a-card code {
+  text-wrap: wrap;
+}

config/config.exs

@@ -6,6 +6,9 @@
 
 import Config
 
+config :chatbot, Chatbot.Repo, types: Chatbot.PostgrexTypes
+config :chatbot, openai_key: "your openai API key"
+config :nx, default_backend: EXLA.Backend
 import_config "config/endpoint.exs"
 import_config "config/logger.exs"
 import_config "config/phoenix.exs"

config/dev.exs

@@ -0,0 +1 @@
+import Config

config/prod.exs

@@ -0,0 +1 @@
+import Config

config/test.exs

@@ -0,0 +1 @@
+import Config

eval/rag_triad_eval.exs

@@ -0,0 +1,63 @@
+openai_key = Application.compile_env(:chatbot, :openai_key)
+
+dataset =
+  "https://huggingface.co/datasets/explodinggradients/amnesty_qa/resolve/main/english.json"
+
+IO.puts("downloading dataset")
+
+data =
+  Req.get!(dataset).body
+  |> Jason.decode!()
+
+IO.puts("indexing")
+
+data["contexts"]
+|> Enum.map(&Enum.join(&1, " "))
+|> Enum.with_index(fn context, index -> %{document: context, source: "#{index}"} end)
+|> Chatbot.Rag.index()
+
+IO.puts("generating responses")
+
+generations =
+  for question <- data["question"] do
+    Chatbot.Rag.query(question)
+  end
+
+provider = Rag.Ai.OpenAI.new(%{text_model: "gpt-4o-mini", api_key: openai_key})
+
+IO.puts("evaluating")
+
+generations =
+  for generation <- generations do
+    Rag.Evaluation.evaluate_rag_triad(generation, provider)
+  end
+
+json =
+  generations
+  |> Enum.map(fn generation ->
+    Map.from_struct(generation)
+    |> Map.take([:query, :context, :context_sources, :response, :evaluations])
+  end)
+  |> Jason.encode!()
+
+File.write!(Path.join(__DIR__, "triad_eval.json"), json)
+
+average_rag_triad_scores =
+  Enum.map(
+    generations,
+    fn gen ->
+      %{
+        evaluations: %{
+          "context_relevance_score" => context_relevance_score,
+          "groundedness_score" => groundedness_score,
+          "answer_relevance_score" => answer_relevance_score
+        }
+      } = gen
+
+      (context_relevance_score + groundedness_score + answer_relevance_score) / 3
+    end
+  )
+
+total_average_score = Enum.sum(average_rag_triad_scores) / Enum.count(average_rag_triad_scores)
+
+IO.puts("Score: ,#{total_average_score}")

lib/chatbot/application.ex

@@ -5,9 +5,18 @@ defmodule Chatbot.Application do
 
   use Application
 
+  alias Chatbot.Rag.Serving
+  alias Chatbot.Rag.TelemetryHandler
+
   @impl true
   def start(_type, _args) do
     children = [
+      {Nx.Serving,
+       [
+         serving: Serving.build_embedding_serving(),
+         name: Rag.EmbeddingServing,
+         batch_timeout: 100
+       ]},
       {Task.Supervisor, name: Chatbot.TaskSupervisor},
       ChatbotWeb.Telemetry,
       Chatbot.Repo,
@@ -19,6 +28,14 @@ defmodule Chatbot.Application do
       ChatbotWeb.Endpoint
     ]
 
+    :ok =
+      :telemetry.attach_many(
+        "rag-handler",
+        Rag.Telemetry.events(),
+        &TelemetryHandler.handle_event/4,
+        nil
+      )
+
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Chatbot.Supervisor]

lib/chatbot/chat.ex

@@ -3,13 +3,12 @@ defmodule Chatbot.Chat do
   Context for chat related functions.
   """
   import Ecto.Query, only: [from: 2]
-  alias Chatbot.{Chat.Message, LLMMock, Repo}
+  alias Chatbot.{Chat.Message, Repo}
   alias LangChain.Chains.LLMChain
   # There is currently a bug in the LangChain type specs:
   # `add_callback/2` expects a map with all possible handler functions.
   # See:
   # https://hexdocs.pm/langchain/0.3.0-rc.0/LangChain.Chains.ChainCallbacks.html#t:chain_callback_handler/0
-  @dialyzer {:nowarn_function, stream_assistant_message: 1}
 
   @doc """
   Creates a message.
@@ -42,16 +41,14 @@ defmodule Chatbot.Chat do
   @spec request_assistant_message([Message.t()]) ::
           {:ok, Message.t()} | {:error, String.t() | Ecto.Changeset.t()}
   def request_assistant_message(messages) do
-    maybe_mock_llm()
-
     messages = Enum.map(messages, &to_langchain_message/1)
 
     @chain
     |> LLMChain.add_messages(messages)
     |> LLMChain.run()
     |> case do
-      {:ok, _chain, response} ->
-        create_message(%{role: :assistant, content: response.content})
+      {:ok, chain} ->
+        create_message(%{role: :assistant, content: chain.last_message.content})
 
       _error ->
         {:error, "I failed, I'm sorry"}
@@ -64,15 +61,13 @@ defmodule Chatbot.Chat do
 
   Once the full message was processed, it is saved as an assistant message.
   """
-  @spec stream_assistant_message(pid()) :: Message.t()
-  def stream_assistant_message(receiver) do
-    messages = all_messages() |> Enum.map(&to_langchain_message/1)
-
-    {:ok, assistant_message} = create_message(%{role: :assistant, content: ""})
+  @spec stream_assistant_message(pid(), [Message.t()], Message.t()) :: Message.t()
+  def stream_assistant_message(receiver, messages, assistant_message) do
+    messages = Enum.map(messages, &to_langchain_message/1)
 
     handler = %{
       on_llm_new_delta: fn _model, %LangChain.MessageDelta{} = data ->
-        send(receiver, {:next_message_delta, assistant_message.id, data})
+        send(receiver, {:next_message_delta, assistant_message, data})
       end,
       on_message_processed: fn _chain, %LangChain.Message{} = data ->
         completed_message = update_message!(assistant_message, %{content: data.content})
@@ -81,29 +76,21 @@ defmodule Chatbot.Chat do
       end
     }
 
-    Task.Supervisor.start_child(Chatbot.TaskSupervisor, fn ->
-      maybe_mock_llm(stream: true)
-
+    {:ok, _chain} =
       @chain
       |> LLMChain.add_callback(handler)
-      |> LLMChain.add_llm_callback(handler)
       |> LLMChain.add_messages(messages)
       |> LLMChain.run()
-    end)
 
     assistant_message
   end
 
-  defp to_langchain_message(%{role: :user, content: content}),
+  def to_langchain_message(%{role: :user, content: content}),
     do: LangChain.Message.new_user!(content)
 
-  defp to_langchain_message(%{role: :assistant, content: content}),
+  def to_langchain_message(%{role: :assistant, content: content}),
     do: LangChain.Message.new_assistant!(content)
 
-  defp maybe_mock_llm(opts \\ []) do
-    if Application.fetch_env!(:chatbot, :mock_llm_api), do: LLMMock.mock(opts)
-  end
-
   @doc """
   Lists all messages ordered by insertion date.
   """

lib/chatbot/chat/message.ex

@@ -13,6 +13,7 @@ defmodule Chatbot.Chat.Message do
   schema "messages" do
     field :role, Ecto.Enum, values: @message_types
     field :content, :string
+    field :sources, {:array, :string}
 
     timestamps()
   end
@@ -24,7 +25,7 @@ defmodule Chatbot.Chat.Message do
   @spec changeset(t(), map()) :: Ecto.Changeset.t()
   def changeset(message \\ %__MODULE__{}, attrs) do
     message
-    |> cast(attrs, [:role])
+    |> cast(attrs, [:role, :sources])
     |> cast(attrs, [:content], empty_values: [nil])
     # we cannot require the content, as
     # validate_required still considers "" as empty