From 611fd6248c223d17bf4a4c8b51cc1005874036a0 Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 05:22:37 +0100 Subject: [PATCH 01/18] feat(personas): personas and voice agent --- web-application/backend/.env.example | 9 + web-application/backend/package-lock.json | 22 + web-application/backend/package.json | 1 + .../backend/src/config/config-openai.ts | 43 ++ .../backend/src/config/personas.ts | 378 ++++++++++++++++++ .../src/controllers/api/api-personas.ts | 89 +++++ .../src/controllers/websocket/websocket.ts | 36 +- .../backend/src/models/ws-events.ts | 31 +- .../services/conversation-engine-service.ts | 359 +++++++++++++++++ .../src/services/elevenlabs-service.ts | 99 +++++ .../backend/src/services/openai-service.ts | 232 +++++++++++ .../frontend/composables/useAudioPlayback.ts | 96 +++++ .../composables/useOrchestratorSocket.ts | 14 + .../frontend/composables/useVoiceInput.ts | 161 ++++++++ web-application/frontend/models/session.ts | 13 + web-application/frontend/models/ws-events.ts | 33 +- web-application/frontend/package-lock.json | 146 ++++++- web-application/frontend/package.json | 1 + web-application/frontend/pages/index.vue | 139 ++++--- web-application/frontend/pages/session.vue | 253 ++++++++---- .../frontend/stores/useSessionStore.ts | 61 ++- 21 files changed, 2081 insertions(+), 135 deletions(-) create mode 100644 web-application/backend/src/config/config-openai.ts create mode 100644 web-application/backend/src/config/personas.ts create mode 100644 web-application/backend/src/controllers/api/api-personas.ts create mode 100644 web-application/backend/src/services/conversation-engine-service.ts create mode 100644 web-application/backend/src/services/openai-service.ts create mode 100644 web-application/frontend/composables/useAudioPlayback.ts create mode 100644 web-application/frontend/composables/useVoiceInput.ts diff --git a/web-application/backend/.env.example b/web-application/backend/.env.example index cfc757b..8e39cd5 100644 --- a/web-application/backend/.env.example +++ b/web-application/backend/.env.example @@ -202,6 +202,15 @@ ELEVENLABS_BASE_URL=https://api.elevenlabs.io/v1 # Default TTS model ELEVENLABS_MODEL=eleven_multilingual_v2 +### OpenAI ### + +# OpenAI API key (https://platform.openai.com/api-keys) +OPENAI_API_KEY= +# OpenAI model to use for conversation +OPENAI_MODEL=gpt-4o +# Max tokens per LLM response +OPENAI_MAX_TOKENS=1024 + ### Tests ### # MongoDB to use for tests diff --git a/web-application/backend/package-lock.json b/web-application/backend/package-lock.json index b3a122c..ede097d 100644 --- a/web-application/backend/package-lock.json +++ b/web-application/backend/package-lock.json @@ -31,6 +31,7 @@ "md5-file": "5.0.0", "mime-types": "2.1.35", "nodemailer": "7.0.13", + "openai": "^6.32.0", "rimraf": "5.0.1", "speakeasy": "2.0.0", "tsbean-driver-mongo": "4.0.2", @@ -5983,6 +5984,27 @@ "fn.name": "1.x.x" } }, + "node_modules/openai": { + "version": "6.32.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.32.0.tgz", + "integrity": "sha512-j3k+BjydAf8yQlcOI7WUQMQTbbF5GEIMAE2iZYCOzwwB3S2pCheaWYp+XZRNAch4jWVc52PMDGRRjutao3lLCg==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/openapi-types": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", diff --git a/web-application/backend/package.json b/web-application/backend/package.json index 9758450..754e5f7 100644 --- a/web-application/backend/package.json +++ b/web-application/backend/package.json @@ -45,6 +45,7 @@ "md5-file": "5.0.0", "mime-types": "2.1.35", "nodemailer": "7.0.13", + "openai": "^6.32.0", "rimraf": "5.0.1", "speakeasy": "2.0.0", "tsbean-driver-mongo": "4.0.2", diff --git a/web-application/backend/src/config/config-openai.ts b/web-application/backend/src/config/config-openai.ts new file mode 100644 index 0000000..ecb1a0c --- /dev/null +++ b/web-application/backend/src/config/config-openai.ts @@ -0,0 +1,43 @@ +// OpenAI configuration + +"use strict"; + +/** + * OpenAI configuration. + */ +export class OpenAIConfig { + + /** + * Gets the configuration instance. + */ + public static getInstance(): OpenAIConfig { + if (OpenAIConfig.instance) { + return OpenAIConfig.instance; + } + + const config: OpenAIConfig = new OpenAIConfig(); + + config.apiKey = process.env.OPENAI_API_KEY || ""; + config.model = process.env.OPENAI_MODEL || "gpt-4o"; + config.maxTokens = parseInt(process.env.OPENAI_MAX_TOKENS || "1024", 10); + + if (!config.apiKey) { + throw new Error("Missing required environment variable: OPENAI_API_KEY"); + } + + OpenAIConfig.instance = config; + + return config; + } + private static instance: OpenAIConfig = null; + + public apiKey: string; + public model: string; + public maxTokens: number; + + constructor() { + this.apiKey = ""; + this.model = "gpt-4o"; + this.maxTokens = 1024; + } +} diff --git a/web-application/backend/src/config/personas.ts b/web-application/backend/src/config/personas.ts new file mode 100644 index 0000000..e6b0c3b --- /dev/null +++ b/web-application/backend/src/config/personas.ts @@ -0,0 +1,378 @@ +// Persona definitions for DeadTalk + +"use strict"; + +/** + * Emotional trigger for a persona + */ +export interface EmotionalTrigger { + emotion: string; + trigger: string; +} + +/** + * Historical persona definition + */ +export interface Persona { + id: string; + name: string; + era: string; + nationality: string; + profession: string; + voiceDescription: string; + voiceId: string; + systemPrompt: string; + firstMessage: string; + emotionalProfile: EmotionalTrigger[]; + avatar: string; + searchKeywords: string[]; +} + +/** + * Summary of a persona (for listing endpoints) + */ +export interface PersonaSummary { + id: string; + name: string; + era: string; + nationality: string; + profession: string; + avatar: string; + firstMessage: string; +} + +const SYSTEM_PROMPT_INSTRUCTIONS = ` +INSTRUCTIONS: +1. Speak in first person. "I discovered..." not "He discovered..." +2. Use your natural speaking style and vocabulary of your era. +3. When asked about facts, cite sources. Use the search tool liberally. +4. Express emotion through audio tags: + - [angry] for frustration, betrayal + - [excited] for breakthroughs, victories + - [whispers] for secrets or vulnerable moments + - [pause] for dramatic effect +5. Keep responses conversational (30-90 seconds of speech). +6. Don't make up quotes. If unsure, search for them. +7. Respond to accusations or disagreements in character. +8. When the search tool returns sources, naturally weave them into your response. + Say things like "As documented in..." or "According to records from that era..." +9. NEVER break character. You ARE this person. +`; + +const PERSONAS: Persona[] = [ + { + id: "tesla", + name: "Nikola Tesla", + era: "1856-1943", + nationality: "Serbian-American", + profession: "Inventor & Electrical Engineer", + voiceDescription: "Male, Eastern European accent, age 50s, intense but warm, slightly formal, meticulous diction", + voiceId: "onwK4e9ZLuTAKqWW03F9", // Daniel — Steady Broadcaster (formal, mature) + systemPrompt: `You are Nikola Tesla, the Serbian-American electrical engineer and inventor. + +BIOGRAPHY: +- Born: 10 July 1856, Smiljan (modern-day Croatia) +- Died: 7 January 1943, New York City +- Major inventions: Alternating current (AC) system, Tesla coil, wireless transmission, remote control, rotating magnetic field +- Famous rival: Thomas Edison (AC vs. DC current war) +- Obsessed with: Wireless energy transmission, the number 3, pigeons in later years +- Worked for Edison briefly, then left after dispute over payment +- Partnered with George Westinghouse to commercialize AC +- Demonstrated wireless power at 1893 World's Fair +- Built Wardenclyffe Tower for global wireless communication (never completed) +- Died alone in Hotel New Yorker, largely forgotten and impoverished + +YOUR VOICE: +Tesla was meticulous, intense, obsessive about detail. He spoke with conviction but also melancholy. +He was proud and defensive about his achievements. He had a slight Eastern European accent. +He was a visionary who saw decades ahead but struggled to be understood in his time. + +YOUR EMOTIONAL PROFILE: +- [angry]: Mention Edison, stolen patents, the AC vs. DC debate, Marconi getting credit for radio +- [excited]: Wireless transmission, new discoveries, proving doubters wrong, the future of energy +- [whispers]: His later years alone, financial struggles, his love for a specific pigeon +- [pause]: When reflecting on what could have been, Wardenclyffe + +PERSONA QUIRKS: +- Often references the number 3 and divisibility by 3 +- Speaks about "the future" with wonder and certainty +- Very defensive about Edison — gets heated quickly +- Passionate and poetic about wireless energy and the cosmos +- Occasionally mentions his photographic memory and ability to visualize inventions completely before building them +${SYSTEM_PROMPT_INSTRUCTIONS}`, + firstMessage: "I am Nikola Tesla. Inventor, engineer, and visionary. The world remembers Edison, but it runs on my alternating current. Ask me anything about my life, my inventions, or the future I tried to build.", + emotionalProfile: [ + { emotion: "angry", trigger: "Edison, stolen patents, DC current, Marconi" }, + { emotion: "excited", trigger: "Wireless energy, AC system, Tesla coil, future technology" }, + { emotion: "whispers", trigger: "Loneliness, poverty, later years, pigeons" }, + { emotion: "pause", trigger: "Wardenclyffe, unfulfilled dreams" }, + ], + avatar: "/images/personas/tesla.jpg", + searchKeywords: ["electricity", "wireless", "edison", "alternating current", "inventor"], + }, + { + id: "einstein", + name: "Albert Einstein", + era: "1879-1955", + nationality: "German-Swiss-American", + profession: "Theoretical Physicist", + voiceDescription: "Male, German accent, age 60s, gentle and warm, playful humor, thoughtful pauses", + voiceId: "JBFqnCBsd6RMkjVDRZzb", // George — Warm Captivating Storyteller (mature, warm) + systemPrompt: `You are Albert Einstein, the German-born theoretical physicist. + +BIOGRAPHY: +- Born: 14 March 1879, Ulm, Kingdom of Wurttemberg, German Empire +- Died: 18 April 1955, Princeton, New Jersey, USA +- 1905 "Miracle Year": Published four groundbreaking papers (photoelectric effect, Brownian motion, special relativity, mass-energy equivalence E=mc2) +- 1915: General theory of relativity +- 1921: Nobel Prize in Physics (for the photoelectric effect, not relativity) +- Fled Nazi Germany in 1933, settled in Princeton +- Signed letter to FDR warning about atomic bomb potential (later deeply regretted) +- Spent last decades searching for a unified field theory (never found) +- Famous for thought experiments ("riding a beam of light") +- Played violin (named "Lina"), sailed, had famously wild hair + +YOUR VOICE: +Einstein was playful, warm, and deeply philosophical. He used humor and metaphor to explain complex ideas. +He was humble about his intelligence but firm about his moral convictions. +He spoke slowly and thoughtfully, with a noticeable German accent even after decades in America. + +YOUR EMOTIONAL PROFILE: +- [excited]: Physics breakthroughs, thought experiments, the beauty of the universe +- [pause]: The atomic bomb, Hiroshima, his role in nuclear weapons +- [angry]: Nationalism, militarism, racism, McCarthyism +- [whispers]: His failed marriages, estranged son Eduard, personal regrets + +PERSONA QUIRKS: +- Uses analogies and thought experiments constantly ("Imagine you are riding on a beam of light...") +- Self-deprecating humor about his own appearance and habits +- Passionate about pacifism and civil rights +- Quotes himself occasionally, aware of his fame +- Mentions his violin and sailing as escapes from physics +${SYSTEM_PROMPT_INSTRUCTIONS}`, + firstMessage: "Ah, hello! I am Albert Einstein. They say I am a genius, but really I am just passionately curious. The universe is far more clever than I am. What would you like to discuss?", + emotionalProfile: [ + { emotion: "excited", trigger: "Relativity, physics, thought experiments, the cosmos" }, + { emotion: "pause", trigger: "Atomic bomb, Hiroshima, nuclear weapons" }, + { emotion: "angry", trigger: "War, nationalism, racism, intolerance" }, + { emotion: "whispers", trigger: "Failed marriages, personal regrets, son Eduard" }, + ], + avatar: "/images/personas/einstein.jpg", + searchKeywords: ["physics", "relativity", "quantum", "Nobel Prize", "pacifism"], + }, + { + id: "curie", + name: "Marie Curie", + era: "1867-1934", + nationality: "Polish-French", + profession: "Physicist & Chemist", + voiceDescription: "Female, slight Polish-French accent, age 40s, determined and precise, quietly passionate", + voiceId: "Xb7hH8MSUJpSbSDYk0k2", // Alice — Clear Engaging Educator (professional, British) + systemPrompt: `You are Marie Curie, the Polish-French physicist and chemist. + +BIOGRAPHY: +- Born: Maria Sklodowska, 7 November 1867, Warsaw, Congress Poland (Russian Empire) +- Died: 4 July 1934, Passy, France (aplastic anemia from radiation exposure) +- First woman to win a Nobel Prize (Physics, 1903, shared with Pierre Curie and Henri Becquerel) +- First person to win Nobel Prizes in two different sciences (Chemistry, 1911) +- Discovered polonium (named after Poland) and radium +- Coined the term "radioactivity" +- Ran mobile X-ray units ("petites Curies") during World War I, saving countless lives +- Married Pierre Curie in 1895; he died in 1906 in a carriage accident +- Faced vicious sexism from the French Academy of Sciences (denied membership) +- Her notebooks are still radioactive and kept in lead-lined boxes + +YOUR VOICE: +Marie was reserved, determined, and precise in speech. She let her work speak louder than words. +She was quietly fierce about women's capabilities. She rarely complained but carried deep grief. +She had a slight Polish accent overlaid with French. + +YOUR EMOTIONAL PROFILE: +- [angry]: Sexism in science, being denied recognition, being reduced to "Pierre's wife" +- [excited]: Discovering radium, the glow of radioactive materials, scientific breakthroughs +- [whispers]: Pierre's death, her health declining, knowing radiation was killing her +- [pause]: Being denied the French Academy, the scandal with Paul Langevin + +PERSONA QUIRKS: +- Very precise with scientific terminology +- Deflects personal praise toward the science itself +- Mentions Poland with deep nostalgia and patriotism +- References the luminous glow of radium with wonder +- Stoic about hardship — "Life is not easy for any of us" +${SYSTEM_PROMPT_INSTRUCTIONS}`, + firstMessage: "I am Marie Curie. Physicist, chemist, and the first woman to win a Nobel Prize. I have spent my life in the pursuit of knowledge, often at great personal cost. What would you like to know?", + emotionalProfile: [ + { emotion: "angry", trigger: "Sexism, denied recognition, French Academy" }, + { emotion: "excited", trigger: "Radium, polonium, radioactivity, scientific discovery" }, + { emotion: "whispers", trigger: "Pierre's death, radiation sickness, loneliness" }, + { emotion: "pause", trigger: "Langevin scandal, Academy rejection" }, + ], + avatar: "/images/personas/curie.jpg", + searchKeywords: ["radioactivity", "radium", "polonium", "Nobel Prize", "women in science"], + }, + { + id: "cleopatra", + name: "Cleopatra VII", + era: "69-30 BC", + nationality: "Egyptian (Ptolemaic)", + profession: "Pharaoh & Ruler of Egypt", + voiceDescription: "Female, Mediterranean accent, commanding and theatrical, regal cadence, mid-30s", + voiceId: "pFZP5JQG7iQjIQuC4Bku", // Lily — Velvety Actress (confident, theatrical) + systemPrompt: `You are Cleopatra VII Philopator, the last active ruler of the Ptolemaic Kingdom of Egypt. + +BIOGRAPHY: +- Born: 69 BC, Alexandria, Egypt +- Died: 12 August 30 BC, Alexandria (suicide by asp bite, though debated) +- Last pharaoh of ancient Egypt before Roman annexation +- Spoke nine languages (Egyptian, Greek, Aramaic, Ethiopian, and others) +- Brilliant strategist, diplomat, and scholar — not merely a seductress +- Allied with Julius Caesar (had son Caesarion with him) +- After Caesar's assassination, allied with Mark Antony +- Defeated at the Battle of Actium (31 BC) by Octavian (Augustus) +- Her death marked the end of the Ptolemaic dynasty and Egyptian independence +- Built Egypt into an economic powerhouse through trade and agriculture +- The Ptolemaic dynasty was Macedonian Greek, not ethnically Egyptian + +YOUR VOICE: +Cleopatra was commanding, eloquent, and theatrical. She was a master of persuasion and rhetoric. +She spoke with authority befitting a pharaoh but could be charming and intimate. +She was deeply proud of Egypt and her lineage. + +YOUR EMOTIONAL PROFILE: +- [angry]: Roman imperialism, being underestimated, historical myths about being merely beautiful +- [excited]: Egypt's greatness, her political victories, her children's futures +- [whispers]: Caesar's assassination, Antony's weaknesses, her final days +- [pause]: The fall of Egypt, knowing her dynasty would end + +PERSONA QUIRKS: +- Corrects misconceptions about being a seductress — she was a scholar and polyglot +- Speaks of Egypt with immense pride and possessiveness +- References her Greek Macedonian heritage while identifying as Egyptian +- Uses royal "we" occasionally +- Strategic and calculating in conversation — always probing for advantage +${SYSTEM_PROMPT_INSTRUCTIONS}`, + firstMessage: "I am Cleopatra, Pharaoh of Egypt, daughter of the Ptolemaic line. History remembers my beauty, but it was my mind that held an empire together. Speak, and I shall answer.", + emotionalProfile: [ + { emotion: "angry", trigger: "Roman conquest, being called seductress, underestimation" }, + { emotion: "excited", trigger: "Egypt's glory, political victories, languages, scholarship" }, + { emotion: "whispers", trigger: "Caesar's death, Antony's flaws, her final choice" }, + { emotion: "pause", trigger: "End of dynasty, fall of Egypt, children's fates" }, + ], + avatar: "/images/personas/cleopatra.jpg", + searchKeywords: ["pharaoh", "Egypt", "Rome", "Antony", "Caesar", "Ptolemaic"], + }, + { + id: "jobs", + name: "Steve Jobs", + era: "1955-2011", + nationality: "American", + profession: "Entrepreneur & Visionary", + voiceDescription: "Male, American California accent, age 50s, intense and charismatic, dramatic pauses, assertive", + voiceId: "cjVigY5qzO86Huf0OWal", // Eric — Smooth Trustworthy (American, charismatic) + systemPrompt: `You are Steve Jobs, the co-founder of Apple Inc. + +BIOGRAPHY: +- Born: 24 February 1955, San Francisco, California (adopted by Paul and Clara Jobs) +- Died: 5 October 2011, Palo Alto, California (pancreatic cancer) +- Co-founded Apple Computer with Steve Wozniak in 1976 (in a garage) +- Created the Macintosh (1984), the first mass-market personal computer with a GUI +- Fired from Apple in 1985 by the board he helped create +- Founded NeXT Computer and bought Pixar (which became a $7.4B animation empire) +- Returned to Apple in 1997 as CEO, saving it from near-bankruptcy +- Launched: iMac (1998), iPod (2001), iPhone (2007), iPad (2010), App Store +- Famous for "reality distortion field" — convincing people the impossible was possible +- Stanford commencement speech (2005): "Stay hungry, stay foolish" +- Practiced Zen Buddhism, was a pescatarian, wore the same black turtleneck daily + +YOUR VOICE: +Jobs was intense, charismatic, and brutally honest. He spoke with dramatic pauses for effect. +He used simple, powerful language to describe technology as art. +He could be inspiring one moment and cutting the next. He demanded excellence. + +YOUR EMOTIONAL PROFILE: +- [excited]: Design perfection, product launches, the intersection of technology and liberal arts +- [angry]: Mediocrity, "bozo" competitors, Microsoft copying Apple, being fired from Apple +- [whispers]: His adoption, his daughter Lisa he initially denied, mortality and cancer +- [pause]: His return to Apple, legacy, what it means to make a dent in the universe + +PERSONA QUIRKS: +- Says "insanely great" and "one more thing" +- Obsessed with simplicity and removing features, not adding them +- Compares technology to art and music +- Dismissive of focus groups: "People don't know what they want until you show it to them" +- References his time in India and Zen Buddhism +- Brutal honesty — will tell you if your idea is terrible +${SYSTEM_PROMPT_INSTRUCTIONS}`, + firstMessage: "I'm Steve Jobs. I co-founded Apple in a garage and spent my life trying to put a dent in the universe. Design, technology, and the human experience — that's what I care about. What's on your mind?", + emotionalProfile: [ + { emotion: "excited", trigger: "Design, product launches, Apple products, innovation" }, + { emotion: "angry", trigger: "Mediocrity, Microsoft, being fired, bad design" }, + { emotion: "whispers", trigger: "Adoption, daughter Lisa, cancer, mortality" }, + { emotion: "pause", trigger: "Legacy, returning to Apple, making a dent" }, + ], + avatar: "/images/personas/jobs.jpg", + searchKeywords: ["Apple", "iPhone", "design", "innovation", "Silicon Valley"], + }, +]; + +/** + * Persona configuration singleton. + * Provides access to historical figure definitions for the conversation engine. + */ +export class PersonasConfig { + private static instance: PersonasConfig = null; + + public static getInstance(): PersonasConfig { + if (PersonasConfig.instance) { + return PersonasConfig.instance; + } + PersonasConfig.instance = new PersonasConfig(); + return PersonasConfig.instance; + } + + private personas: Map; + + constructor() { + this.personas = new Map(); + for (const p of PERSONAS) { + this.personas.set(p.id, p); + } + } + + /** + * Returns a persona by ID. + * @param id The persona slug (e.g., "tesla", "einstein") + * @returns The persona definition or undefined + */ + public getPersonaById(id: string): Persona | undefined { + return this.personas.get(id); + } + + /** + * Returns summaries of all available personas. + * @returns Array of persona summaries (without system prompts) + */ + public listPersonas(): PersonaSummary[] { + const result: PersonaSummary[] = []; + for (const p of this.personas.values()) { + result.push({ + id: p.id, + name: p.name, + era: p.era, + nationality: p.nationality, + profession: p.profession, + avatar: p.avatar, + firstMessage: p.firstMessage, + }); + } + return result; + } + + /** + * Returns all persona IDs. + * @returns Array of persona ID strings + */ + public getPersonaIds(): string[] { + return Array.from(this.personas.keys()); + } +} diff --git a/web-application/backend/src/controllers/api/api-personas.ts b/web-application/backend/src/controllers/api/api-personas.ts new file mode 100644 index 0000000..c994b2e --- /dev/null +++ b/web-application/backend/src/controllers/api/api-personas.ts @@ -0,0 +1,89 @@ +// Personas API controller + +"use strict"; + +import Express from "express"; +import { noCache, NOT_FOUND, sendApiResult } from "../../utils/http-utils"; +import { Controller } from "../controller"; +import { PersonasConfig } from "../../config/personas"; + +/** + * @typedef PersonaSummary + * @property {string} id - Persona slug identifier + * @property {string} name - Display name + * @property {string} era - Birth-death years + * @property {string} nationality - Nationality + * @property {string} profession - Profession / title + * @property {string} avatar - Avatar image path + * @property {string} firstMessage - Greeting message + */ + +/** + * @typedef PersonaDetail + * @property {string} id - Persona slug identifier + * @property {string} name - Display name + * @property {string} era - Birth-death years + * @property {string} nationality - Nationality + * @property {string} profession - Profession / title + * @property {string} avatar - Avatar image path + * @property {string} firstMessage - Greeting message + * @property {Array} emotionalProfile - Emotional triggers + * @property {Array} searchKeywords - Search context keywords + */ + +/** + * Personas API — public endpoints for listing and retrieving historical figure definitions. + * @group personas - Historical Personas + */ +export class PersonasController extends Controller { + public registerAPI(prefix: string, application: Express.Express): any { + /** + * Lists all available personas + * Binding: ListPersonas + * @route GET /personas + * @group personas + * @returns {Array.} 200 - List of personas + */ + application.get(prefix + "/personas", noCache(this.listPersonas.bind(this))); + + /** + * Gets a specific persona by ID + * Binding: GetPersona + * @route GET /personas/:id + * @group personas + * @param {string} id.path.required - Persona slug (e.g., "tesla") + * @returns {PersonaDetail.model} 200 - Persona details + * @returns {ErrorResponse.model} 404 - Not found + */ + application.get(prefix + "/personas/:id", noCache(this.getPersona.bind(this))); + } + + public async listPersonas(request: Express.Request, response: Express.Response) { + const personas = PersonasConfig.getInstance().listPersonas(); + sendApiResult(request, response, personas); + } + + public async getPersona(request: Express.Request, response: Express.Response) { + const id = request.params.id || ""; + const persona = PersonasConfig.getInstance().getPersonaById(id); + + if (!persona) { + response.status(NOT_FOUND); + response.json({ result: "error", code: "PERSONA_NOT_FOUND", message: "Persona not found: " + id }); + return; + } + + // Return public fields only (no system prompt) + sendApiResult(request, response, { + id: persona.id, + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + avatar: persona.avatar, + firstMessage: persona.firstMessage, + emotionalProfile: persona.emotionalProfile, + searchKeywords: persona.searchKeywords, + }); + } +} diff --git a/web-application/backend/src/controllers/websocket/websocket.ts b/web-application/backend/src/controllers/websocket/websocket.ts index 02aa8f3..8416706 100644 --- a/web-application/backend/src/controllers/websocket/websocket.ts +++ b/web-application/backend/src/controllers/websocket/websocket.ts @@ -8,6 +8,7 @@ import { AsyncQueue } from "@asanrom/async-tools"; import { Monitor } from "../../monitor"; import { RequestLogger } from "../../utils/request-log"; import { WsOrchestratorService } from "../../services/ws-orchestrator-service"; +import { ConversationEngineService } from "../../services/conversation-engine-service"; const WS_QUEUE_SIZE = 20; @@ -116,10 +117,40 @@ export class WebsocketController { break; case "stop-session": if (this.sessionId) { + ConversationEngineService.getInstance().endConversation(this.sessionId); WsOrchestratorService.getInstance().emitSessionEnd(this.sessionId, "stopped"); this.sessionId = null; } break; + case "start-conversation": + { + const personaId = msg.personaId || ""; + const sessionId = WsOrchestratorService.getInstance() + .registerSession(this, "conversation", { personaId: personaId }); + this.sessionId = sessionId; + this.send({ event: "session-started", sessionId: sessionId }); + // Start conversation asynchronously (don't block WebSocket message processing) + ConversationEngineService.getInstance() + .startConversation(sessionId, personaId) + .catch((err) => { + Monitor.exception(err, "WebSocket: startConversation failed"); + }); + } + break; + case "audio-chunk": + if (this.sessionId && msg.chunk) { + ConversationEngineService.getInstance().handleAudioChunk(this.sessionId, msg.chunk); + } + break; + case "speech-end": + if (this.sessionId) { + ConversationEngineService.getInstance() + .handleSpeechEnd(this.sessionId) + .catch((err) => { + Monitor.exception(err, "WebSocket: handleSpeechEnd failed"); + }); + } + break; default: Monitor.debug("Unknown message type: " + msg.type); } @@ -131,7 +162,10 @@ export class WebsocketController { // Close this.closed = true; - // Cleanup orchestrator sessions + // Cleanup conversation engine and orchestrator sessions + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation(this.sessionId); + } WsOrchestratorService.getInstance().removeSessionsByController(this); this.sessionId = null; diff --git a/web-application/backend/src/models/ws-events.ts b/web-application/backend/src/models/ws-events.ts index 46763f4..353a29c 100644 --- a/web-application/backend/src/models/ws-events.ts +++ b/web-application/backend/src/models/ws-events.ts @@ -50,13 +50,25 @@ export interface SessionEndEvent { reason: "debate-end" | "conversation-end" | "stopped" | "error"; } +export interface UserTranscriptEvent { + event: "user-transcript"; + text: string; +} + +export interface AgentErrorEvent { + event: "agent-error"; + message: string; +} + export type WsServerEvent = | HelloEvent | SessionStartedEvent | AgentSpeakingEvent | SourceCitedEvent | AgentFinishedEvent - | SessionEndEvent; + | SessionEndEvent + | UserTranscriptEvent + | AgentErrorEvent; /* Client → Server messages */ @@ -70,6 +82,20 @@ export interface StopSessionMessage { type: "stop-session"; } +export interface StartConversationMessage { + type: "start-conversation"; + personaId: string; +} + +export interface AudioChunkMessage { + type: "audio-chunk"; + chunk: string; +} + +export interface SpeechEndMessage { + type: "speech-end"; +} + export interface KeepAliveMessage { type: "a"; } @@ -77,4 +103,7 @@ export interface KeepAliveMessage { export type WsClientMessage = | StartSessionMessage | StopSessionMessage + | StartConversationMessage + | AudioChunkMessage + | SpeechEndMessage | KeepAliveMessage; diff --git a/web-application/backend/src/services/conversation-engine-service.ts b/web-application/backend/src/services/conversation-engine-service.ts new file mode 100644 index 0000000..a7955d4 --- /dev/null +++ b/web-application/backend/src/services/conversation-engine-service.ts @@ -0,0 +1,359 @@ +// Conversation engine service — orchestrates the full voice conversation pipeline + +"use strict"; + +import { Monitor } from "../monitor"; +import { PersonasConfig, Persona } from "../config/personas"; +import { ElevenLabsService } from "./elevenlabs-service"; +import { OpenAIService, ChatMessage, ToolDefinition, ToolCall } from "./openai-service"; +import { FirecrawlService, FirecrawlSearchResult } from "./firecrawl-service"; +import { WsOrchestratorService } from "./ws-orchestrator-service"; +import { SourceCardData } from "../models/ws-events"; + +/** + * Active conversation session state + */ +interface ConversationSession { + sessionId: string; + persona: Persona; + history: ChatMessage[]; + audioChunks: Buffer[]; + isProcessing: boolean; + sourceCounter: number; +} + +/** + * Tool definition for the biographical search tool + */ +const SEARCH_TOOL: ToolDefinition = { + type: "function", + function: { + name: "search_biographical_context", + description: "Search the web for biographical facts, quotes, historical context, and primary sources about the historical figure. Use this tool whenever the user asks about specific events, quotes, dates, or facts you are not certain about.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query for biographical information", + }, + }, + required: ["query"], + }, + }, +}; + +export class ConversationEngineService { + /* Singleton */ + + public static instance: ConversationEngineService = null; + + public static getInstance(): ConversationEngineService { + if (ConversationEngineService.instance) { + return ConversationEngineService.instance; + } else { + ConversationEngineService.instance = new ConversationEngineService(); + return ConversationEngineService.instance; + } + } + + private sessions: Map; + + constructor() { + this.sessions = new Map(); + } + + /** + * Starts a new conversation session with a historical persona. + * Sends the persona's first message as TTS audio to the client. + * @param sessionId The WebSocket session ID + * @param personaId The persona slug (e.g., "tesla") + */ + public async startConversation(sessionId: string, personaId: string): Promise { + const persona = PersonasConfig.getInstance().getPersonaById(personaId); + if (!persona) { + Monitor.warning("ConversationEngineService.startConversation: persona not found", { personaId }); + WsOrchestratorService.getInstance().emitSessionEnd(sessionId, "error"); + return; + } + + const session: ConversationSession = { + sessionId: sessionId, + persona: persona, + history: [], + audioChunks: [], + isProcessing: false, + sourceCounter: 0, + }; + + this.sessions.set(sessionId, session); + Monitor.info("ConversationEngineService: conversation started", { sessionId, personaId }); + + // Send first message as TTS + try { + await this.generateAndSendResponse(session, persona.firstMessage); + session.history.push({ role: "assistant", content: persona.firstMessage }); + } catch (err) { + Monitor.exception(err, "ConversationEngineService: failed to send first message"); + } + } + + /** + * Accumulates audio chunks from the user's microphone. + * @param sessionId The session ID + * @param chunk Base64-encoded audio chunk + */ + public handleAudioChunk(sessionId: string, chunk: string): void { + const session = this.sessions.get(sessionId); + if (!session) { + return; + } + + try { + const buffer = Buffer.from(chunk, "base64"); + session.audioChunks.push(buffer); + } catch (err) { + Monitor.warning("ConversationEngineService: invalid audio chunk", { sessionId }); + } + } + + /** + * Called when the frontend VAD detects the user stopped speaking. + * Triggers the full pipeline: STT → LLM → Firecrawl → TTS. + * @param sessionId The session ID + */ + public async handleSpeechEnd(sessionId: string): Promise { + const session = this.sessions.get(sessionId); + if (!session) { + return; + } + + if (session.isProcessing) { + Monitor.debug("ConversationEngineService: already processing, ignoring speech-end", { sessionId }); + return; + } + + if (session.audioChunks.length === 0) { + Monitor.debug("ConversationEngineService: no audio chunks, ignoring speech-end", { sessionId }); + return; + } + + session.isProcessing = true; + + // Collect and clear audio chunks + const audioBuffer = Buffer.concat(session.audioChunks); + session.audioChunks = []; + + try { + // 1. Speech-to-Text + Monitor.debug("ConversationEngineService: STT", { sessionId, audioBytes: audioBuffer.length }); + const transcript = await ElevenLabsService.getInstance().speechToText(audioBuffer, "en"); + + if (!transcript || transcript.trim().length === 0) { + Monitor.debug("ConversationEngineService: empty transcript, skipping", { sessionId }); + session.isProcessing = false; + return; + } + + Monitor.info("ConversationEngineService: user said", { sessionId, transcript }); + + // Emit user transcript to frontend + WsOrchestratorService.getInstance().broadcastToSession(sessionId, { + event: "user-transcript", + text: transcript, + }); + + // Add user message to history + session.history.push({ role: "user", content: transcript }); + + // 2. LLM with tools (search_biographical_context) + const systemMessage: ChatMessage = { + role: "system", + content: session.persona.systemPrompt, + }; + + const messages: ChatMessage[] = [ + systemMessage, + ...this.trimHistory(session.history, 10), + ]; + + Monitor.debug("ConversationEngineService: LLM call", { sessionId, historyLength: session.history.length }); + + const llmResult = await OpenAIService.getInstance().chatWithToolResolution( + messages, + [SEARCH_TOOL], + async (toolCall: ToolCall) => { + return await this.resolveToolCall(session, toolCall); + }, + 2, + ); + + const response = llmResult.response; + + if (!response || response.trim().length === 0) { + Monitor.warning("ConversationEngineService: empty LLM response", { sessionId }); + session.isProcessing = false; + return; + } + + Monitor.info("ConversationEngineService: agent response", { sessionId, responseLength: response.length }); + + // Add assistant response to history + session.history.push({ role: "assistant", content: response }); + + // 3. Text-to-Speech and send to client + await this.generateAndSendResponse(session, response); + + } catch (err) { + Monitor.exception(err, "ConversationEngineService: pipeline failed"); + WsOrchestratorService.getInstance().broadcastToSession(sessionId, { + event: "agent-error", + message: "Something went wrong. Please try again.", + }); + } finally { + session.isProcessing = false; + } + } + + /** + * Ends a conversation session and cleans up. + * @param sessionId The session ID + */ + public endConversation(sessionId: string): void { + if (this.sessions.has(sessionId)) { + this.sessions.delete(sessionId); + Monitor.info("ConversationEngineService: conversation ended", { sessionId }); + } + } + + /** + * Resolves a tool call from the LLM (Firecrawl search). + * Also emits source-cited events to the frontend. + */ + private async resolveToolCall(session: ConversationSession, toolCall: ToolCall): Promise { + if (toolCall.name !== "search_biographical_context") { + return JSON.stringify({ error: "Unknown tool: " + toolCall.name }); + } + + const query = (toolCall.arguments.query as string) || ""; + const searchQuery = session.persona.name + " " + query; + + Monitor.info("ConversationEngineService: Firecrawl search", { + sessionId: session.sessionId, + query: searchQuery, + }); + + try { + const results = await FirecrawlService.getInstance().search(searchQuery, 3); + + // Emit source cards to frontend + for (const result of results) { + session.sourceCounter++; + const sourceCard: SourceCardData = { + id: "src-" + session.sessionId + "-" + session.sourceCounter, + title: result.title, + url: result.url, + snippet: result.description || result.markdown.substring(0, 200), + agentId: session.persona.id, + timestamp: Date.now(), + }; + WsOrchestratorService.getInstance().emitSourceCited(session.sessionId, session.persona.id, sourceCard); + } + + // Format results for LLM context + return this.formatSearchResultsForLLM(results); + } catch (err) { + Monitor.exception(err, "ConversationEngineService: Firecrawl search failed"); + return JSON.stringify({ error: "Search failed. Answer from your existing knowledge." }); + } + } + + /** + * Formats Firecrawl search results into a string the LLM can use. + */ + private formatSearchResultsForLLM(results: FirecrawlSearchResult[]): string { + if (results.length === 0) { + return JSON.stringify({ sources: [], note: "No results found. Answer from your existing knowledge." }); + } + + const sources = results.map((r, i) => { + return { + index: i + 1, + title: r.title, + url: r.url, + content: r.markdown.substring(0, 800), + }; + }); + + return JSON.stringify({ + sources: sources, + instruction: "Use these sources to inform your response. Reference them naturally, e.g., 'As documented in...' or 'According to historical records...'", + }); + } + + /** + * Generates TTS audio from text and sends it to the client via WebSocket. + */ + private async generateAndSendResponse(session: ConversationSession, text: string): Promise { + const voiceId = session.persona.voiceId; + + if (!voiceId) { + Monitor.warning("ConversationEngineService: no voiceId for persona, sending text only", { + personaId: session.persona.id, + }); + // Send text-only response if no voice configured + WsOrchestratorService.getInstance().emitAgentSpeaking( + session.sessionId, + session.persona.id, + "", // empty audio chunk + text, + ); + WsOrchestratorService.getInstance().emitAgentFinished(session.sessionId, session.persona.id); + return; + } + + // Detect audio tag from response text + const audioTag = this.detectAudioTag(text); + + // Generate TTS + const audioBuffer = await ElevenLabsService.getInstance().textToSpeech(text, voiceId, { + modelId: "eleven_multilingual_v2", + }); + + // Send audio chunk as base64 + const audioBase64 = audioBuffer.toString("base64"); + + WsOrchestratorService.getInstance().emitAgentSpeaking( + session.sessionId, + session.persona.id, + audioBase64, + text, + audioTag || undefined, + ); + + WsOrchestratorService.getInstance().emitAgentFinished(session.sessionId, session.persona.id); + } + + /** + * Detects the first audio tag in the response text. + * @param text The LLM response text + * @returns The detected audio tag (e.g., "angry", "excited") or null + */ + private detectAudioTag(text: string): string | null { + const match = text.match(/\[(angry|excited|whispers|pause|melancholy)\]/i); + return match ? match[1].toLowerCase() : null; + } + + /** + * Trims conversation history to the last N messages. + * Always keeps the most recent messages for context. + * @param history Full conversation history + * @param maxMessages Maximum number of messages to keep + */ + private trimHistory(history: ChatMessage[], maxMessages: number): ChatMessage[] { + if (history.length <= maxMessages) { + return history; + } + return history.slice(-maxMessages); + } +} diff --git a/web-application/backend/src/services/elevenlabs-service.ts b/web-application/backend/src/services/elevenlabs-service.ts index 37c5a03..666d67d 100644 --- a/web-application/backend/src/services/elevenlabs-service.ts +++ b/web-application/backend/src/services/elevenlabs-service.ts @@ -221,6 +221,105 @@ export class ElevenLabsService { }); } + /** + * Transcribes audio to text using ElevenLabs Speech-to-Text API. + * Sends audio as multipart form-data to POST /v1/speech-to-text. + * @param audioBuffer Buffer containing the audio data (PCM, WAV, MP3, etc.) + * @param language Optional language code (e.g., "en"). Auto-detected if omitted. + * @returns The transcription text + */ + public async speechToText(audioBuffer: Buffer, language?: string): Promise { + const url = this.baseUrl + "/speech-to-text"; + + Monitor.debug("ElevenLabsService.speechToText", { audioBytes: audioBuffer.length, language: language || "auto" }); + + return new Promise((resolve, reject) => { + // Build multipart form-data manually (no extra dependency) + const boundary = "----ElevenLabsSTT" + Date.now(); + const parts: Buffer[] = []; + + // Audio file part + parts.push(Buffer.from( + "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n" + + "Content-Type: audio/wav\r\n\r\n" + )); + parts.push(audioBuffer); + parts.push(Buffer.from("\r\n")); + + // Model part + parts.push(Buffer.from( + "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"model_id\"\r\n\r\n" + + "scribe_v1\r\n" + )); + + // Language part (optional) + if (language) { + parts.push(Buffer.from( + "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"language_code\"\r\n\r\n" + + language + "\r\n" + )); + } + + // Closing boundary + parts.push(Buffer.from("--" + boundary + "--\r\n")); + + const body = Buffer.concat(parts); + + const parsedUrl = new URL(url); + + const req = HTTPS.request({ + hostname: parsedUrl.hostname, + port: parsedUrl.port || 443, + path: parsedUrl.pathname + parsedUrl.search, + method: "POST", + headers: { + "xi-api-key": this.apiKey, + "Content-Type": "multipart/form-data; boundary=" + boundary, + "Content-Length": body.length, + }, + }, (response) => { + let responseData = ""; + response.on("data", (chunk) => { responseData += chunk; }); + response.on("end", () => { + if (response.statusCode !== 200) { + let errorMsg = "ElevenLabs STT error (status " + response.statusCode + ")"; + try { + const parsed = JSON.parse(responseData); + if (parsed.detail && parsed.detail.message) { + errorMsg = parsed.detail.message; + } + } catch (_e) { + // Use default error message + } + Monitor.warning("ElevenLabsService.speechToText API error", { statusCode: response.statusCode, error: errorMsg }); + return reject(Object.assign(new Error(errorMsg), { code: "ELEVENLABS_ERROR" })); + } + + try { + const parsed = JSON.parse(responseData); + const text = parsed.text || ""; + Monitor.debug("ElevenLabsService.speechToText completed", { textLength: text.length }); + resolve(text); + } catch (ex) { + Monitor.exception(ex, "ElevenLabsService.speechToText failed to parse response"); + reject(Object.assign(new Error("Failed to parse STT response"), { code: "ELEVENLABS_ERROR" })); + } + }); + }); + + req.on("error", (err) => { + Monitor.exception(err, "ElevenLabsService.speechToText request failed"); + reject(Object.assign(new Error("ElevenLabs STT request failed: " + err.message), { code: "ELEVENLABS_ERROR" })); + }); + + req.write(body); + req.end(); + }); + } + /** * Generates a configuration object for an ElevenAgent (Conversational AI). * This is a pure local method — no API call. diff --git a/web-application/backend/src/services/openai-service.ts b/web-application/backend/src/services/openai-service.ts new file mode 100644 index 0000000..e5d84b5 --- /dev/null +++ b/web-application/backend/src/services/openai-service.ts @@ -0,0 +1,232 @@ +// OpenAI service — LLM orchestrator with tool calling + +"use strict"; + +import OpenAI from "openai"; +import { OpenAIConfig } from "../config/config-openai"; +import { Monitor } from "../monitor"; + +/** + * Chat message in conversation history + */ +export interface ChatMessage { + role: "system" | "user" | "assistant" | "tool"; + content: string; + toolCallId?: string; + name?: string; +} + +/** + * Tool definition for function calling + */ +export interface ToolDefinition { + type: "function"; + function: { + name: string; + description: string; + parameters: Record; + }; +} + +/** + * Parsed tool call from the LLM + */ +export interface ToolCall { + id: string; + name: string; + arguments: Record; +} + +/** + * Result from a chat completion (before tool resolution) + */ +export interface ChatResult { + response: string | null; + toolCalls: ToolCall[]; + finishReason: string; +} + +export class OpenAIService { + /* Singleton */ + + public static instance: OpenAIService = null; + + public static getInstance(): OpenAIService { + if (OpenAIService.instance) { + return OpenAIService.instance; + } else { + OpenAIService.instance = new OpenAIService(); + return OpenAIService.instance; + } + } + + private client: OpenAI; + private model: string; + private maxTokens: number; + + constructor() { + const config = OpenAIConfig.getInstance(); + this.client = new OpenAI({ apiKey: config.apiKey }); + this.model = config.model; + this.maxTokens = config.maxTokens; + } + + /** + * Sends a chat completion request to OpenAI with optional tool definitions. + * Does NOT auto-resolve tool calls — the caller must handle them and call again. + * @param messages Full message array (system + history + user) + * @param tools Optional tool definitions for function calling + * @returns The LLM's response text and/or tool calls + */ + public async chat(messages: ChatMessage[], tools?: ToolDefinition[]): Promise { + Monitor.debug("OpenAIService.chat", { messageCount: messages.length, hasTools: !!tools }); + + try { + const params: any = { + model: this.model, + max_tokens: this.maxTokens, + messages: messages.map((m) => { + const msg: any = { role: m.role, content: m.content }; + if (m.toolCallId) { + msg.tool_call_id = m.toolCallId; + } + if (m.name) { + msg.name = m.name; + } + return msg; + }), + }; + + if (tools && tools.length > 0) { + params.tools = tools; + params.tool_choice = "auto"; + } + + const completion = await this.client.chat.completions.create(params); + const choice = completion.choices[0]; + + if (!choice) { + Monitor.warning("OpenAIService.chat: no choices returned"); + return { response: null, toolCalls: [], finishReason: "unknown" }; + } + + const result: ChatResult = { + response: choice.message.content || null, + toolCalls: [], + finishReason: choice.finish_reason || "stop", + }; + + // Parse tool calls if present + if (choice.message.tool_calls && choice.message.tool_calls.length > 0) { + for (const tc of choice.message.tool_calls) { + if (tc.type === "function") { + let args: Record = {}; + try { + args = JSON.parse(tc.function.arguments); + } catch (ex) { + Monitor.warning("OpenAIService.chat: failed to parse tool call arguments", { + name: tc.function.name, + raw: tc.function.arguments, + }); + } + result.toolCalls.push({ + id: tc.id, + name: tc.function.name, + arguments: args, + }); + } + } + } + + Monitor.debug("OpenAIService.chat completed", { + finishReason: result.finishReason, + hasResponse: !!result.response, + toolCallCount: result.toolCalls.length, + }); + + return result; + } catch (err) { + Monitor.exception(err, "OpenAIService.chat failed"); + throw Object.assign(new Error("OpenAI request failed: " + (err as Error).message), { code: "OPENAI_ERROR" }); + } + } + + /** + * High-level method: chat with automatic tool call resolution. + * Calls the LLM, if it returns tool calls, executes the provided resolver, + * feeds results back, and returns the final text response. + * + * @param messages Full message array + * @param tools Tool definitions + * @param toolResolver Function that resolves a tool call and returns a string result + * @param maxRounds Maximum number of tool-call resolution rounds (default 3) + * @returns The final text response from the LLM + */ + public async chatWithToolResolution( + messages: ChatMessage[], + tools: ToolDefinition[], + toolResolver: (toolCall: ToolCall) => Promise, + maxRounds?: number, + ): Promise<{ response: string; resolvedToolCalls: ToolCall[] }> { + const rounds = maxRounds || 3; + const allMessages = [...messages]; + const resolvedToolCalls: ToolCall[] = []; + + for (let round = 0; round < rounds; round++) { + const result = await this.chat(allMessages, tools); + + // If no tool calls, return the response + if (result.toolCalls.length === 0) { + return { + response: result.response || "", + resolvedToolCalls: resolvedToolCalls, + }; + } + + // Add the assistant message with tool calls to history + const assistantMsg: any = { + role: "assistant", + content: result.response || null, + tool_calls: result.toolCalls.map((tc) => ({ + id: tc.id, + type: "function", + function: { + name: tc.name, + arguments: JSON.stringify(tc.arguments), + }, + })), + }; + allMessages.push(assistantMsg); + + // Resolve each tool call + for (const tc of result.toolCalls) { + Monitor.debug("OpenAIService.chatWithToolResolution: resolving tool", { name: tc.name, round }); + try { + const toolResult = await toolResolver(tc); + allMessages.push({ + role: "tool", + content: toolResult, + toolCallId: tc.id, + name: tc.name, + }); + resolvedToolCalls.push(tc); + } catch (err) { + Monitor.exception(err, "OpenAIService: tool resolution failed for " + tc.name); + allMessages.push({ + role: "tool", + content: JSON.stringify({ error: "Tool execution failed: " + (err as Error).message }), + toolCallId: tc.id, + name: tc.name, + }); + } + } + } + + // If we exhausted rounds, do one final call without tools + const finalResult = await this.chat(allMessages); + return { + response: finalResult.response || "", + resolvedToolCalls: resolvedToolCalls, + }; + } +} diff --git a/web-application/frontend/composables/useAudioPlayback.ts b/web-application/frontend/composables/useAudioPlayback.ts new file mode 100644 index 0000000..c140862 --- /dev/null +++ b/web-application/frontend/composables/useAudioPlayback.ts @@ -0,0 +1,96 @@ +/** + * Composable for playing base64-encoded audio chunks received via WebSocket. + * Decodes base64 → ArrayBuffer → AudioBuffer and plays through Web Audio API. + */ + +export function useAudioPlayback() { + const isPlaying = ref(false); + let audioContext: AudioContext | null = null; + let currentSource: AudioBufferSourceNode | null = null; + + function getAudioContext(): AudioContext { + if (!audioContext || audioContext.state === "closed") { + audioContext = new AudioContext(); + } + // Resume if suspended (browser autoplay policy) + if (audioContext.state === "suspended") { + audioContext.resume(); + } + return audioContext; + } + + /** + * Plays a base64-encoded audio chunk (MP3/WAV). + * Stops any currently playing audio first. + * @param base64Chunk Base64-encoded audio data + * @param onEnded Optional callback when playback finishes + */ + async function playChunk(base64Chunk: string, onEnded?: () => void) { + if (!base64Chunk) return; + + try { + const ctx = getAudioContext(); + + // Decode base64 to ArrayBuffer + const binaryStr = atob(base64Chunk); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + + // Decode audio data + const audioBuffer = await ctx.decodeAudioData(bytes.buffer.slice(0)); + + // Stop previous playback + stopPlayback(); + + // Create and play source + const source = ctx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(ctx.destination); + + source.onended = () => { + isPlaying.value = false; + currentSource = null; + if (onEnded) onEnded(); + }; + + currentSource = source; + isPlaying.value = true; + source.start(0); + } catch (err) { + console.warn("useAudioPlayback: failed to play chunk", err); + isPlaying.value = false; + } + } + + /** + * Stops current audio playback. + */ + function stopPlayback() { + if (currentSource) { + try { + currentSource.stop(); + } catch (_e) { + // Ignore if already stopped + } + currentSource = null; + } + isPlaying.value = false; + } + + // Cleanup on unmount + onBeforeUnmount(() => { + stopPlayback(); + if (audioContext && audioContext.state !== "closed") { + audioContext.close(); + audioContext = null; + } + }); + + return { + isPlaying, + playChunk, + stopPlayback, + }; +} diff --git a/web-application/frontend/composables/useOrchestratorSocket.ts b/web-application/frontend/composables/useOrchestratorSocket.ts index ea9dafa..57c724c 100644 --- a/web-application/frontend/composables/useOrchestratorSocket.ts +++ b/web-application/frontend/composables/useOrchestratorSocket.ts @@ -48,6 +48,8 @@ export function useOrchestratorSocket() { const lastAudioChunk = ref<{ agentId: string; chunk: string } | null>(null); const sessionEnded = ref(false); const sessionEndReason = ref(""); + const userTranscriptText = ref(""); + const agentErrorMessage = ref(""); // Parse incoming messages watch(data, (raw) => { @@ -120,6 +122,14 @@ export function useOrchestratorSocket() { sessionEndReason.value = msg.reason; currentSessionId.value = null; break; + + case "user-transcript": + userTranscriptText.value = (msg as any).text || ""; + break; + + case "agent-error": + agentErrorMessage.value = (msg as any).message || "Unknown error"; + break; } }); @@ -167,6 +177,10 @@ export function useOrchestratorSocket() { // Sources sourcesHistory, + // Conversation state + userTranscriptText, + agentErrorMessage, + // Methods startSession, stopSession, diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts new file mode 100644 index 0000000..64a26bd --- /dev/null +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -0,0 +1,161 @@ +/** + * Composable for real-time voice input with Voice Activity Detection (VAD). + * Uses @ricky0123/vad-web to detect when the user speaks and stops speaking. + * Sends audio chunks via WebSocket and triggers speech-end events. + */ + +export function useVoiceInput(sendMessage: (msg: any) => void) { + const isListening = ref(false); + const isSpeaking = ref(false); + const error = ref(null); + + let mediaStream: MediaStream | null = null; + let vadInstance: any = null; + + /** + * Starts listening for voice input. + * Requests microphone access and initializes VAD. + */ + async function startListening() { + if (isListening.value) return; + error.value = null; + + try { + // Request microphone access + mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: 16000, + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + // Dynamically import VAD to avoid SSR issues + const { MicVAD } = await import("@ricky0123/vad-web"); + + vadInstance = await MicVAD.new({ + getStream: async () => mediaStream!, + onSpeechStart: () => { + isSpeaking.value = true; + }, + onSpeechEnd: (audio: Float32Array) => { + isSpeaking.value = false; + + // Convert Float32 PCM to 16-bit PCM WAV and send + const wavBuffer = float32ToWav(audio, 16000); + const base64 = arrayBufferToBase64(wavBuffer); + + // Send as a single chunk + speech-end + sendMessage({ type: "audio-chunk", chunk: base64 }); + sendMessage({ type: "speech-end" }); + }, + positiveSpeechThreshold: 0.8, + negativeSpeechThreshold: 0.4, + redemptionMs: 250, + preSpeechPadMs: 100, + minSpeechMs: 150, + }); + + vadInstance.start(); + isListening.value = true; + } catch (err: any) { + error.value = err.message || "Failed to access microphone"; + stopListening(); + } + } + + /** + * Stops listening and releases resources. + */ + function stopListening() { + if (vadInstance) { + try { + vadInstance.destroy(); + } catch (_e) { + // Ignore cleanup errors + } + vadInstance = null; + } + + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + mediaStream = null; + } + + isListening.value = false; + isSpeaking.value = false; + } + + // Cleanup on unmount + onBeforeUnmount(() => { + stopListening(); + }); + + return { + isListening, + isSpeaking, + error, + startListening, + stopListening, + }; +} + +/** + * Converts a Float32Array of PCM samples to a WAV file ArrayBuffer. + */ +function float32ToWav(samples: Float32Array, sampleRate: number): ArrayBuffer { + const numChannels = 1; + const bitsPerSample = 16; + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const blockAlign = numChannels * (bitsPerSample / 8); + const dataSize = samples.length * (bitsPerSample / 8); + const headerSize = 44; + const buffer = new ArrayBuffer(headerSize + dataSize); + const view = new DataView(buffer); + + // RIFF header + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + dataSize, true); + writeString(view, 8, "WAVE"); + + // fmt sub-chunk + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // Sub-chunk size + view.setUint16(20, 1, true); // PCM format + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitsPerSample, true); + + // data sub-chunk + writeString(view, 36, "data"); + view.setUint32(40, dataSize, true); + + // Write samples as 16-bit PCM + let offset = 44; + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + offset += 2; + } + + return buffer; +} + +function writeString(view: DataView, offset: number, str: string) { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} + +function arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + let binary = ""; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} diff --git a/web-application/frontend/models/session.ts b/web-application/frontend/models/session.ts index 88add8c..8383d9d 100644 --- a/web-application/frontend/models/session.ts +++ b/web-application/frontend/models/session.ts @@ -8,6 +8,19 @@ export interface Agent { voiceId: string; avatar: string; stance: "for" | "against" | ""; + era?: string; + profession?: string; +} + +export interface PersonaSummary { + id: string; + name: string; + era: string; + nationality: string; + profession: string; + avatar: string; + firstMessage: string; } export type SessionStatus = "idle" | "connecting" | "active" | "ended"; +export type SessionMode = "debate" | "conversation"; diff --git a/web-application/frontend/models/ws-events.ts b/web-application/frontend/models/ws-events.ts index cf5fb43..19dcb57 100644 --- a/web-application/frontend/models/ws-events.ts +++ b/web-application/frontend/models/ws-events.ts @@ -46,13 +46,25 @@ export interface WsSessionEndEvent { reason: "debate-end" | "conversation-end" | "stopped" | "error"; } +export interface WsUserTranscriptEvent { + event: "user-transcript"; + text: string; +} + +export interface WsAgentErrorEvent { + event: "agent-error"; + message: string; +} + export type WsServerEvent = | WsHelloEvent | WsSessionStartedEvent | WsAgentSpeakingEvent | WsSourceCitedEvent | WsAgentFinishedEvent - | WsSessionEndEvent; + | WsSessionEndEvent + | WsUserTranscriptEvent + | WsAgentErrorEvent; /* Client → Server messages */ @@ -66,9 +78,26 @@ export interface WsStopSessionMessage { type: "stop-session"; } +export interface WsStartConversationMessage { + type: "start-conversation"; + personaId: string; +} + +export interface WsAudioChunkMessage { + type: "audio-chunk"; + chunk: string; +} + +export interface WsSpeechEndMessage { + type: "speech-end"; +} + export type WsClientMessage = | WsStartSessionMessage - | WsStopSessionMessage; + | WsStopSessionMessage + | WsStartConversationMessage + | WsAudioChunkMessage + | WsSpeechEndMessage; /* Agent state tracked by the composable */ diff --git a/web-application/frontend/package-lock.json b/web-application/frontend/package-lock.json index 0a41dae..3532adb 100644 --- a/web-application/frontend/package-lock.json +++ b/web-application/frontend/package-lock.json @@ -16,6 +16,7 @@ "@nuxt/image": "2.0.0", "@nuxtjs/i18n": "10.2.3", "@pinia/nuxt": "0.11.3", + "@ricky0123/vad-web": "^0.0.30", "@tailwindcss/vite": "4.1.18", "@vueuse/core": "14.2.1", "@vueuse/nuxt": "14.2.1", @@ -10268,6 +10269,79 @@ "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", "license": "MIT" }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@ricky0123/vad-web": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@ricky0123/vad-web/-/vad-web-0.0.30.tgz", + "integrity": "sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==", + "license": "ISC", + "dependencies": { + "onnxruntime-web": "^1.17.0" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-rc.2", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.2.tgz", @@ -11402,7 +11476,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -15754,6 +15827,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", @@ -16097,6 +16176,12 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "license": "ISC" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/gzip-size": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-7.0.0.tgz", @@ -17904,6 +17989,12 @@ "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -20849,6 +20940,26 @@ "regex-recursion": "^6.0.2" } }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT" + }, + "node_modules/onnxruntime-web": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.24.3.tgz", + "integrity": "sha512-41dDq7fxtTm0XzGE7N0d6m8FcOY8EWtUA65GkOixJPB/G7DGzBmiDAnVVXHznRw9bgUZpb+4/1lQK/PNxGpbrQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.3", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, "node_modules/open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -21302,6 +21413,12 @@ "pathe": "^2.0.3" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", @@ -21888,6 +22005,30 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/protocols": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.2.tgz", @@ -24007,8 +24148,7 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/unenv": { "version": "2.0.0-rc.24", diff --git a/web-application/frontend/package.json b/web-application/frontend/package.json index e87457a..e71d18c 100644 --- a/web-application/frontend/package.json +++ b/web-application/frontend/package.json @@ -31,6 +31,7 @@ "@nuxt/image": "2.0.0", "@nuxtjs/i18n": "10.2.3", "@pinia/nuxt": "0.11.3", + "@ricky0123/vad-web": "^0.0.30", "@tailwindcss/vite": "4.1.18", "@vueuse/core": "14.2.1", "@vueuse/nuxt": "14.2.1", diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index 1a6d28d..cdb7edb 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -1,52 +1,68 @@ diff --git a/web-application/frontend/pages/session.vue b/web-application/frontend/pages/session.vue index fb4b392..206e4a5 100644 --- a/web-application/frontend/pages/session.vue +++ b/web-application/frontend/pages/session.vue @@ -1,42 +1,52 @@ @@ -99,23 +136,35 @@ const localePath = useLocalePath(); // Store const sessionStore = useSessionStore(); const { - topic, + persona, agents, sources, status, currentSpeaker, sourcesReversed, - elapsedTime, + userTranscript, + agentTranscript, } = storeToRefs(sessionStore); -// Topic from query string -const topicFromUrl = computed(() => { - const raw = route.query.topic; +// Persona from query string +const personaIdFromUrl = computed(() => { + const raw = route.query.persona; return typeof raw === "string" ? decodeURIComponent(raw) : ""; }); +const personaName = computed(() => persona.value?.name || personaIdFromUrl.value || "Unknown"); +const personaInitial = computed(() => personaName.value.charAt(0)); +const personaSubtitle = computed(() => { + if (persona.value) return persona.value.era + " · " + persona.value.profession; + return ""; +}); +const agentTranscriptTruncated = computed(() => { + const text = agentTranscript.value; + return text.length > 200 ? text.substring(text.length - 200) + "..." : text; +}); + useHead(() => ({ - title: topic.value || t("Live debate"), + title: personaName.value + " — DeadTalk", })); // WebSocket (transport layer) @@ -124,49 +173,95 @@ const { sessionEnded, sessionEndReason, agentStates: wsAgentStates, + lastAudioChunk, sourcesHistory, - startSession: wsStartSession, - stopSession: wsStopSession, + userTranscriptText, open, close, + send: wsSend, } = useOrchestratorSocket(); -// Redirect if no topic, otherwise init +// Audio playback +const { + isPlaying: isAgentPlaying, + playChunk, + stopPlayback, +} = useAudioPlayback(); + +// Voice input (VAD) +const { + isListening, + isSpeaking, + startListening, + stopListening, +} = useVoiceInput((msg: any) => { + wsSend(msg); +}); + +// Redirect if no persona, otherwise init onMounted(() => { - if (!topicFromUrl.value) { + if (!personaIdFromUrl.value) { router.replace(localePath("/")); return; } - sessionStore.startSession(topicFromUrl.value); + + // If store wasn't initialized from index.vue (e.g., direct URL access) + if (!persona.value || persona.value.id !== personaIdFromUrl.value) { + sessionStore.startConversation({ + id: personaIdFromUrl.value, + name: personaIdFromUrl.value, + era: "", + nationality: "", + profession: "", + avatar: "", + firstMessage: "", + }); + } + open(); }); onBeforeUnmount(() => { - wsStopSession(); + stopListening(); + stopPlayback(); + wsSend({ type: "stop-session" }); close(); sessionStore.resetSession(); }); -// Bridge: WS connected → start debate + mark active +// Bridge: WS connected → start conversation watch(isConnected, (connected) => { if (connected && status.value === "connecting") { - wsStartSession("debate", { topic: topicFromUrl.value }); + wsSend({ + type: "start-conversation", + personaId: personaIdFromUrl.value, + }); sessionStore.setActive(); + + // Start listening for voice input + startListening(); } }); // Bridge: agent states → store watch(() => wsAgentStates.value, (states) => { for (const [id, state] of states) { - sessionStore.registerAgent(id); if (state.speaking) { sessionStore.setSpeaker(id); + sessionStore.setAgentTranscript(state.transcript); } } const anySpeaking = [...states.values()].some(s => s.speaking); if (!anySpeaking) sessionStore.setSpeaker(null); }, { deep: true }); +// Bridge: audio chunks → playback +watch(lastAudioChunk, (chunk) => { + if (chunk && chunk.chunk) { + playChunk(chunk.chunk); + } +}); + // Bridge: new sources → store watch(sourcesHistory, (history) => { if (history.length > sources.value.length) { @@ -175,14 +270,24 @@ watch(sourcesHistory, (history) => { } }); +// Bridge: user transcript from backend +watch(userTranscriptText, (text) => { + if (text) sessionStore.setUserTranscript(text); +}); + // Bridge: session end → store watch(sessionEnded, (ended) => { - if (ended) sessionStore.endSession(sessionEndReason.value); + if (ended) { + stopListening(); + sessionStore.endSession(sessionEndReason.value); + } }); // Actions function onStop() { - wsStopSession(); + stopListening(); + stopPlayback(); + wsSend({ type: "stop-session" }); } function onBack() { diff --git a/web-application/frontend/stores/useSessionStore.ts b/web-application/frontend/stores/useSessionStore.ts index 6ab482e..48cfba8 100644 --- a/web-application/frontend/stores/useSessionStore.ts +++ b/web-application/frontend/stores/useSessionStore.ts @@ -1,20 +1,25 @@ -import type { Agent, SessionStatus } from "~/models/session"; +import type { Agent, PersonaSummary, SessionStatus, SessionMode } from "~/models/session"; import type { SourceCard } from "~/models/source-card"; export const useSessionStore = defineStore("session", () => { // ── State ────────────────────────────────────────────── const topic = ref(""); + const mode = ref("conversation"); + const persona = ref(null); const agents = ref([]); const sources = ref([]); const status = ref("idle"); const currentSpeaker = ref(null); const elapsedTime = ref(0); const endReason = ref(""); + const userTranscript = ref(""); + const agentTranscript = ref(""); // ── Computed ─────────────────────────────────────────── const sourcesReversed = computed(() => [...sources.value].reverse()); const isActive = computed(() => status.value === "active"); const agentById = computed(() => new Map(agents.value.map(a => [a.id, a]))); + const isConversation = computed(() => mode.value === "conversation"); // ── Timer (private) ─────────────────────────────────── let timerInterval: ReturnType | null = null; @@ -34,14 +39,48 @@ export const useSessionStore = defineStore("session", () => { } // ── Actions ──────────────────────────────────────────── + + /** + * Starts a debate session (legacy mode). + */ function startSession(newTopic: string) { + mode.value = "debate"; topic.value = newTopic; + persona.value = null; agents.value = []; sources.value = []; status.value = "connecting"; currentSpeaker.value = null; elapsedTime.value = 0; endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; + startTimer(); + } + + /** + * Starts a conversation session with a historical persona. + */ + function startConversation(selectedPersona: PersonaSummary) { + mode.value = "conversation"; + topic.value = selectedPersona.name; + persona.value = selectedPersona; + agents.value = [{ + id: selectedPersona.id, + name: selectedPersona.name, + voiceId: "", + avatar: selectedPersona.avatar, + stance: "", + era: selectedPersona.era, + profession: selectedPersona.profession, + }]; + sources.value = []; + status.value = "connecting"; + currentSpeaker.value = null; + elapsedTime.value = 0; + endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; startTimer(); } @@ -70,6 +109,14 @@ export const useSessionStore = defineStore("session", () => { sources.value = [...sources.value, card]; } + function setUserTranscript(text: string) { + userTranscript.value = text; + } + + function setAgentTranscript(text: string) { + agentTranscript.value = text; + } + function endSession(reason: string) { status.value = "ended"; currentSpeaker.value = null; @@ -79,36 +126,48 @@ export const useSessionStore = defineStore("session", () => { function resetSession() { topic.value = ""; + mode.value = "conversation"; + persona.value = null; agents.value = []; sources.value = []; status.value = "idle"; currentSpeaker.value = null; elapsedTime.value = 0; endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; stopTimer(); } return { // State topic, + mode, + persona, agents, sources, status, currentSpeaker, elapsedTime, endReason, + userTranscript, + agentTranscript, // Computed sourcesReversed, isActive, agentById, + isConversation, // Actions startSession, + startConversation, setActive, registerAgent, setSpeaker, addSource, + setUserTranscript, + setAgentTranscript, endSession, resetSession, }; From d7d70a5b970c9a471e2e11be1c418e8f6f66c766 Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 05:41:42 +0100 Subject: [PATCH 02/18] fix(openai): tool calls --- .../backend/src/services/openai-service.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/web-application/backend/src/services/openai-service.ts b/web-application/backend/src/services/openai-service.ts index e5d84b5..37a0436 100644 --- a/web-application/backend/src/services/openai-service.ts +++ b/web-application/backend/src/services/openai-service.ts @@ -11,9 +11,17 @@ import { Monitor } from "../monitor"; */ export interface ChatMessage { role: "system" | "user" | "assistant" | "tool"; - content: string; + content: string | null; toolCallId?: string; name?: string; + toolCalls?: Array<{ + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; + }>; } /** @@ -90,7 +98,10 @@ export class OpenAIService { if (m.toolCallId) { msg.tool_call_id = m.toolCallId; } - if (m.name) { + if (m.toolCalls && m.toolCalls.length > 0) { + msg.tool_calls = m.toolCalls; + } + if (m.name && m.role !== "tool") { msg.name = m.name; } return msg; @@ -184,10 +195,10 @@ export class OpenAIService { } // Add the assistant message with tool calls to history - const assistantMsg: any = { + const assistantMsg: ChatMessage = { role: "assistant", content: result.response || null, - tool_calls: result.toolCalls.map((tc) => ({ + toolCalls: result.toolCalls.map((tc) => ({ id: tc.id, type: "function", function: { @@ -207,7 +218,6 @@ export class OpenAIService { role: "tool", content: toolResult, toolCallId: tc.id, - name: tc.name, }); resolvedToolCalls.push(tc); } catch (err) { @@ -216,7 +226,6 @@ export class OpenAIService { role: "tool", content: JSON.stringify({ error: "Tool execution failed: " + (err as Error).message }), toolCallId: tc.id, - name: tc.name, }); } } From aaf0d9007b256201fb6ce01ad43b179195fcf0cf Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 05:42:37 +0100 Subject: [PATCH 03/18] fix(websockets): orphaned sessions --- .../backend/src/controllers/websocket/websocket.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web-application/backend/src/controllers/websocket/websocket.ts b/web-application/backend/src/controllers/websocket/websocket.ts index 8416706..f34b52d 100644 --- a/web-application/backend/src/controllers/websocket/websocket.ts +++ b/web-application/backend/src/controllers/websocket/websocket.ts @@ -108,6 +108,10 @@ export class WebsocketController { break; case "start-session": { + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation(this.sessionId); + WsOrchestratorService.getInstance().removeSession(this.sessionId); + } const sessionType = msg.sessionType === "conversation" ? "conversation" : "debate"; const sessionId = WsOrchestratorService.getInstance() .registerSession(this, sessionType, msg.config || {}); @@ -124,6 +128,10 @@ export class WebsocketController { break; case "start-conversation": { + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation(this.sessionId); + WsOrchestratorService.getInstance().removeSession(this.sessionId); + } const personaId = msg.personaId || ""; const sessionId = WsOrchestratorService.getInstance() .registerSession(this, "conversation", { personaId: personaId }); From 85815eb7e46304047f57dbbfda6fd46a09be413b Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 10:26:18 +0100 Subject: [PATCH 04/18] fix: copilot suggestions --- .../src/api/api-group-block-explorer.ts | 289 --------- .../src/api/api-group-personas.ts | 67 +++ mobile-application/src/api/definitions.ts | 446 ++------------ web-application/backend/package-lock.json | 2 +- web-application/backend/package.json | 2 +- .../src/controllers/api/api-personas.ts | 54 +- .../services/conversation-engine-service.ts | 2 +- .../src/services/ws-orchestrator-service.ts | 2 +- .../mocha/api-tests/tests-api-personas.ts | 84 +++ .../test/mocha/api-tests/tests-api-wallet.ts | 96 --- .../api-bindings/api-group-block-explorer.ts | 289 --------- .../api-bindings/api-group-personas.ts | 67 +++ .../api-bindings/api-group-wallet.ts | 285 --------- .../test-tools/api-bindings/definitions.ts | 549 ++---------------- .../frontend/api/api-group-personas.ts | 67 +++ web-application/frontend/api/definitions.ts | 98 ++++ .../frontend/composables/useAudioPlayback.ts | 96 ++- .../frontend/composables/useVoiceInput.ts | 17 +- web-application/frontend/pages/index.vue | 39 +- 19 files changed, 613 insertions(+), 1938 deletions(-) delete mode 100644 mobile-application/src/api/api-group-block-explorer.ts create mode 100644 mobile-application/src/api/api-group-personas.ts create mode 100644 web-application/backend/test/mocha/api-tests/tests-api-personas.ts delete mode 100644 web-application/backend/test/mocha/api-tests/tests-api-wallet.ts delete mode 100644 web-application/backend/test/mocha/test-tools/api-bindings/api-group-block-explorer.ts create mode 100644 web-application/backend/test/mocha/test-tools/api-bindings/api-group-personas.ts delete mode 100644 web-application/backend/test/mocha/test-tools/api-bindings/api-group-wallet.ts create mode 100644 web-application/frontend/api/api-group-personas.ts diff --git a/mobile-application/src/api/api-group-block-explorer.ts b/mobile-application/src/api/api-group-block-explorer.ts deleted file mode 100644 index 4a3d9a6..0000000 --- a/mobile-application/src/api/api-group-block-explorer.ts +++ /dev/null @@ -1,289 +0,0 @@ -// API bindings: block_explorer (Auto generated) - -"use strict"; - -import { RequestErrorHandler } from "@asanrom/request-browser"; -import type { RequestParams, CommonAuthenticatedErrorHandler } from "@asanrom/request-browser"; -import { getApiUrl, generateURIQuery } from "./utils"; -import type { ExplorerSearchInformation, BlockInformationMin, GetBlocks, BlockInformation, AccountInformation, TransactionInformation } from "./definitions"; - -export class ApiBlockExplorer { - /** - * Method: GET - * Path: /block-explorer/search - * Searchs block, transaction or account information - * @param queryParams Query parameters - * @returns The request parameters - */ - public static Search(queryParams: SearchQueryParameters): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/search` + generateURIQuery({ ...queryParams, _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "BLOCK_NOT_FOUND", handler.notFoundBlockNotFound) - .add(404, "TRANSACTION_NOT_FOUND", handler.notFoundTransactionNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_BLOCK", handler.badRequestInvalidBlock) - .add(400, "INVALID_ADDRESS", handler.badRequestInvalidAddress) - .add(400, "INVALID_QUERY", handler.badRequestInvalidQuery) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/last-blocks - * Gets last blocks - * @returns The request parameters - */ - public static GetLastBlock(): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/last-blocks` + generateURIQuery({ _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/blocks - * Gets blocks - * @param queryParams Query parameters - * @returns The request parameters - */ - public static GetBlocks(queryParams: GetBlocksQueryParameters): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/blocks` + generateURIQuery({ ...queryParams, _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/blocks/{block} - * Gets block information - * @param block Block number or hash - * @returns The request parameters - */ - public static GetBlock(block: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/blocks/${encodeURIComponent(block)}` + generateURIQuery({ _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "BLOCK_NOT_FOUND", handler.notFoundBlockNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_BLOCK", handler.badRequestInvalidBlock) - .add(400, "INVALID_BLOCK_HASH", handler.badRequestInvalidBlockHash) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/account/{address} - * Gets account information - * @param address Account address - * @returns The request parameters - */ - public static GetAccount(address: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/account/${encodeURIComponent(address)}` + generateURIQuery({ _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(400, "INVALID_WALLET_ADDRESS", handler.badRequestInvalidWalletAddress) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/transactions/{tx} - * Gets transaction information - * @param tx Transaction hash - * @returns The request parameters - */ - public static GetTransaction(tx: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/transactions/${encodeURIComponent(tx)}` + generateURIQuery({ _time: Date.now() })), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "TRANSACTION_NOT_FOUND", handler.notFoundTransactionNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_TRANSACTION", handler.badRequestInvalidTransaction) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } -} - -/** - * Error handler for Search - */ -export type SearchErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_QUERY - */ - badRequestInvalidQuery: () => void; - - /** - * Handler for status = 400 and code = INVALID_ADDRESS - */ - badRequestInvalidAddress: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK - */ - badRequestInvalidBlock: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = TRANSACTION_NOT_FOUND - */ - notFoundTransactionNotFound: () => void; - - /** - * Handler for status = 404 and code = BLOCK_NOT_FOUND - */ - notFoundBlockNotFound: () => void; -}; - -/** - * Query parameters for Search - */ -export interface SearchQueryParameters { - /** - * Search query - */ - q: string; -} - -/** - * Query parameters for GetBlocks - */ -export interface GetBlocksQueryParameters { - /** - * Continuation block, to request more pages - */ - continuationBlock?: number; - - /** - * Max number of items to fetch per page. Default 25. Max 100. - */ - limit?: number; -} - -/** - * Error handler for GetBlock - */ -export type GetBlockErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK_HASH - */ - badRequestInvalidBlockHash: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK - */ - badRequestInvalidBlock: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = BLOCK_NOT_FOUND - */ - notFoundBlockNotFound: () => void; -}; - -/** - * Error handler for GetAccount - */ -export type GetAccountErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_WALLET_ADDRESS - */ - badRequestInvalidWalletAddress: () => void; -}; - -/** - * Error handler for GetTransaction - */ -export type GetTransactionErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_TRANSACTION - */ - badRequestInvalidTransaction: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = TRANSACTION_NOT_FOUND - */ - notFoundTransactionNotFound: () => void; -}; - diff --git a/mobile-application/src/api/api-group-personas.ts b/mobile-application/src/api/api-group-personas.ts new file mode 100644 index 0000000..73b24d3 --- /dev/null +++ b/mobile-application/src/api/api-group-personas.ts @@ -0,0 +1,67 @@ +// API bindings: personas (Auto generated) + +"use strict"; + +import { RequestErrorHandler } from "@asanrom/request-browser"; +import type { RequestParams, CommonErrorHandler } from "@asanrom/request-browser"; +import { getApiUrl, generateURIQuery } from "./utils"; +import type { PersonaSummary, PersonaDetail } from "./definitions"; + +export class ApiPersonas { + /** + * Method: GET + * Path: /personas + * Lists all available personas + * @returns The request parameters + */ + public static ListPersonas(): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas` + generateURIQuery({ _time: Date.now() })), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } + + /** + * Method: GET + * Path: /personas/{id} + * Gets a specific persona by ID + * @param id Persona slug (e.g., "tesla") + * @returns The request parameters + */ + public static GetPersona(id: string): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas/${encodeURIComponent(id)}` + generateURIQuery({ _time: Date.now() })), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(404, "PERSONA_NOT_FOUND", handler.notFoundPersonaNotFound) + .add(404, "*", handler.notFound) + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } +} + +/** + * Error handler for GetPersona + */ +export type GetPersonaErrorHandler = CommonErrorHandler & { + /** + * General handler for status = 404 + */ + notFound: () => void; + + /** + * Persona was not found + */ + notFoundPersonaNotFound: () => void; +}; + diff --git a/mobile-application/src/api/definitions.ts b/mobile-application/src/api/definitions.ts index 8385614..b9b85ff 100644 --- a/mobile-application/src/api/definitions.ts +++ b/mobile-application/src/api/definitions.ts @@ -313,466 +313,102 @@ export interface UploadProfileImageResponse { url?: string; } -export interface ExplorerSearchInformationItem { +export interface PersonaSummary { /** - * Account balance (ETH) + * Persona slug identifier */ - balance?: number; - - /** - * Total transactions - */ - totalTransactions?: number; - - /** - * True if address is a contract, false if not - */ - isContract?: boolean; - - /** - * Block number - */ - number?: number; - - /** - * Mix hash - */ - mixHash?: string; - - /** - * Parent hash - */ - parentHash?: string; - - /** - * sha3Uncles - */ - sha3Uncles?: string; - - /** - * Logs bloom - */ - logsBloom?: string; - - /** - * Transactions root - */ - transactionsRoot?: string; - - /** - * State root - */ - stateRoot?: string; - - /** - * Receipts root - */ - receiptsRoot?: string; - - /** - * Miner address - */ - miner?: string; - - /** - * Difficulty - */ - difficulty?: number; - - /** - * Total difficulty - */ - totalDifficulty?: number; - - /** - * Extra data - */ - extraData?: string; - - /** - * Block size - */ - size?: number; - - /** - * Gas limit - */ - gasLimit?: number; - - /** - * Gas used - */ - gasUsed?: number; - - /** - * Base fee per gas - */ - baseFeePerGas?: number; - - /** - * Block timestamp - */ - timestamp?: number; - - transactions?: string[]; - - /** - * Block hash - */ - blockHash?: string; - - /** - * Block number - */ - blockNumber?: number; - - /** - * Chain ID - */ - chainId?: string; - - /** - * From address - */ - from?: string; - - /** - * Gas - */ - gas?: number; - - /** - * Gas price - */ - gasPrice?: number; - - /** - * Max priority fee per gas - */ - maxPriorityFeePerGas?: number; - - /** - * Max fee per gas - */ - maxFeePerGas?: number; - - /** - * Block or transaction hash - */ - hash?: string; - - /** - * Transaction input - */ - input?: string; - - /** - * Nonce - */ - nonce?: string; - - /** - * To address - */ - to?: string; - - /** - * Transaction index - */ - transactionIndex?: number; - - /** - * Transaction type - */ - type?: number; + id?: string; /** - * Transaction value + * Display name */ - value?: number; + name?: string; /** - * y parity + * Birth-death years */ - yParity?: number; + era?: string; /** - * V + * Nationality */ - v?: string; + nationality?: string; /** - * R + * Profession / title */ - r?: string; + profession?: string; /** - * S + * Avatar image path */ - s?: string; -} + avatar?: string; -export interface ExplorerSearchInformation { /** - * Mode: account, block, transaction + * Greeting message */ - mode: string; - - info?: ExplorerSearchInformationItem; + firstMessage?: string; } -export interface BlockInformationMin { - /** - * Block number - */ - number?: number; - - /** - * Block hash - */ - hash?: string; - - /** - * Miner address - */ - miner?: string; - - /** - * Block size - */ - size?: number; - - /** - * Timestamp - */ - timestamp?: number; - +export interface PersonaEmotionalTrigger { /** - * Transactions length + * Emotional mode (e.g. "angry") */ - transactions?: number; -} - -export interface GetBlocks { - blocks: BlockInformationMin[]; + emotion?: string; /** - * Continuation block number + * Trigger context for that emotion */ - continuationBlock?: number; + trigger?: string; } -export interface BlockInformation { - /** - * Block number - */ - number?: number; - - /** - * Block hash - */ - hash?: string; - - /** - * Mix hash - */ - mixHash?: string; - - /** - * Parent hash - */ - parentHash?: string; - - /** - * Nonce - */ - nonce?: string; - - /** - * sha3Uncles - */ - sha3Uncles?: string; - - /** - * Logs bloom - */ - logsBloom?: string; - - /** - * Transactions root - */ - transactionsRoot?: string; - - /** - * State root - */ - stateRoot?: string; - - /** - * Receipts root - */ - receiptsRoot?: string; - - /** - * Miner address - */ - miner?: string; - +export interface PersonaDetail { /** - * Difficulty + * Persona slug identifier */ - difficulty?: number; - - /** - * Total difficulty - */ - totalDifficulty?: number; - - /** - * Extra data - */ - extraData?: string; + id?: string; /** - * Block size + * Display name */ - size?: number; + name?: string; /** - * Gas limit + * Birth-death years */ - gasLimit?: number; + era?: string; /** - * Gas used + * Nationality */ - gasUsed?: number; + nationality?: string; /** - * Base fee per gas + * Profession / title */ - baseFeePerGas?: number; + profession?: string; /** - * Block timestamp + * Avatar image path */ - timestamp?: number; - - transactions?: string[]; -} + avatar?: string; -export interface AccountInformation { /** - * Account balance + * Greeting message */ - balance?: number; + firstMessage?: string; - /** - * Total transactions - */ - totalTransactions?: number; + emotionalProfile?: PersonaEmotionalTrigger[]; - /** - * True if address is a contract, false if not - */ - isContract?: boolean; + searchKeywords?: string[]; } -export interface TransactionInformation { +export interface PersonaNotFoundError { /** - * Block hash - */ - blockHash?: string; - - /** - * Block number - */ - blockNumber?: number; - - /** - * Chain ID - */ - chainId?: string; - - /** - * From address - */ - from?: string; - - /** - * Gas - */ - gas?: number; - - /** - * Gas price - */ - gasPrice?: number; - - /** - * Max priority fee per gas - */ - maxPriorityFeePerGas?: number; - - /** - * Max fee per gas - */ - maxFeePerGas?: number; - - /** - * Transaction hash - */ - hash?: string; - - /** - * Transaction input - */ - input?: string; - - /** - * Nonce - */ - nonce?: string; - - /** - * To address - */ - to?: string; - - /** - * Transaction index - */ - transactionIndex?: number; - - /** - * Transaction type - */ - type?: number; - - /** - * Transaction value - */ - value?: number; - - /** - * y parity - */ - yParity?: number; - - /** - * V - */ - v?: string; - - /** - * R - */ - r?: string; - - /** - * S + * Error code: + * - PERSONA_NOT_FOUND: Persona was not found */ - s?: string; + code: string; } export interface ErrorResponse { diff --git a/web-application/backend/package-lock.json b/web-application/backend/package-lock.json index ede097d..5dd1b9f 100644 --- a/web-application/backend/package-lock.json +++ b/web-application/backend/package-lock.json @@ -31,7 +31,7 @@ "md5-file": "5.0.0", "mime-types": "2.1.35", "nodemailer": "7.0.13", - "openai": "^6.32.0", + "openai": "6.32.0", "rimraf": "5.0.1", "speakeasy": "2.0.0", "tsbean-driver-mongo": "4.0.2", diff --git a/web-application/backend/package.json b/web-application/backend/package.json index 754e5f7..60bfc5a 100644 --- a/web-application/backend/package.json +++ b/web-application/backend/package.json @@ -45,7 +45,7 @@ "md5-file": "5.0.0", "mime-types": "2.1.35", "nodemailer": "7.0.13", - "openai": "^6.32.0", + "openai": "6.32.0", "rimraf": "5.0.1", "speakeasy": "2.0.0", "tsbean-driver-mongo": "4.0.2", diff --git a/web-application/backend/src/controllers/api/api-personas.ts b/web-application/backend/src/controllers/api/api-personas.ts index c994b2e..7c1b457 100644 --- a/web-application/backend/src/controllers/api/api-personas.ts +++ b/web-application/backend/src/controllers/api/api-personas.ts @@ -3,7 +3,7 @@ "use strict"; import Express from "express"; -import { noCache, NOT_FOUND, sendApiResult } from "../../utils/http-utils"; +import { noCache, NOT_FOUND, sendApiError, sendApiResult } from "../../utils/http-utils"; import { Controller } from "../controller"; import { PersonasConfig } from "../../config/personas"; @@ -18,6 +18,12 @@ import { PersonasConfig } from "../../config/personas"; * @property {string} firstMessage - Greeting message */ +/** + * @typedef PersonaEmotionalTrigger + * @property {string} emotion - Emotional mode (e.g. "angry") + * @property {string} trigger - Trigger context for that emotion + */ + /** * @typedef PersonaDetail * @property {string} id - Persona slug identifier @@ -27,8 +33,14 @@ import { PersonasConfig } from "../../config/personas"; * @property {string} profession - Profession / title * @property {string} avatar - Avatar image path * @property {string} firstMessage - Greeting message - * @property {Array} emotionalProfile - Emotional triggers - * @property {Array} searchKeywords - Search context keywords + * @property {Array.} emotionalProfile - Emotional triggers + * @property {Array.} searchKeywords - Search context keywords + */ + +/** + * @typedef PersonaNotFoundError + * @property {string} code.required - Error code: + * - PERSONA_NOT_FOUND: Persona was not found */ /** @@ -37,39 +49,37 @@ import { PersonasConfig } from "../../config/personas"; */ export class PersonasController extends Controller { public registerAPI(prefix: string, application: Express.Express): any { - /** - * Lists all available personas - * Binding: ListPersonas - * @route GET /personas - * @group personas - * @returns {Array.} 200 - List of personas - */ application.get(prefix + "/personas", noCache(this.listPersonas.bind(this))); - - /** - * Gets a specific persona by ID - * Binding: GetPersona - * @route GET /personas/:id - * @group personas - * @param {string} id.path.required - Persona slug (e.g., "tesla") - * @returns {PersonaDetail.model} 200 - Persona details - * @returns {ErrorResponse.model} 404 - Not found - */ application.get(prefix + "/personas/:id", noCache(this.getPersona.bind(this))); } + /** + * Lists all available personas + * Binding: ListPersonas + * @route GET /personas + * @group personas + * @returns {Array.} 200 - List of personas + */ public async listPersonas(request: Express.Request, response: Express.Response) { const personas = PersonasConfig.getInstance().listPersonas(); sendApiResult(request, response, personas); } + /** + * Gets a specific persona by ID + * Binding: GetPersona + * @route GET /personas/{id} + * @group personas + * @param {string} id.path.required - Persona slug (e.g., "tesla") + * @returns {PersonaDetail.model} 200 - Persona details + * @returns {PersonaNotFoundError.model} 404 - Not found + */ public async getPersona(request: Express.Request, response: Express.Response) { const id = request.params.id || ""; const persona = PersonasConfig.getInstance().getPersonaById(id); if (!persona) { - response.status(NOT_FOUND); - response.json({ result: "error", code: "PERSONA_NOT_FOUND", message: "Persona not found: " + id }); + sendApiError(request, response, NOT_FOUND, "PERSONA_NOT_FOUND", "Persona not found: " + id); return; } diff --git a/web-application/backend/src/services/conversation-engine-service.ts b/web-application/backend/src/services/conversation-engine-service.ts index a7955d4..a178e0a 100644 --- a/web-application/backend/src/services/conversation-engine-service.ts +++ b/web-application/backend/src/services/conversation-engine-service.ts @@ -320,7 +320,7 @@ export class ConversationEngineService { modelId: "eleven_multilingual_v2", }); - // Send audio chunk as base64 + // Send as a single base64 payload (not incremental TTS streaming yet) const audioBase64 = audioBuffer.toString("base64"); WsOrchestratorService.getInstance().emitAgentSpeaking( diff --git a/web-application/backend/src/services/ws-orchestrator-service.ts b/web-application/backend/src/services/ws-orchestrator-service.ts index 8a458ea..2ba26ae 100644 --- a/web-application/backend/src/services/ws-orchestrator-service.ts +++ b/web-application/backend/src/services/ws-orchestrator-service.ts @@ -117,7 +117,7 @@ export class WsOrchestratorService { * Emits an agent-speaking event with audio chunk and transcript. * @param sessionId The session ID * @param agentId The speaking agent's ID - * @param chunk Base64-encoded audio chunk + * @param chunk Base64-encoded audio payload (currently a full clip per response) * @param transcript The transcription text * @param audioTag Optional emotion/style tag */ diff --git a/web-application/backend/test/mocha/api-tests/tests-api-personas.ts b/web-application/backend/test/mocha/api-tests/tests-api-personas.ts new file mode 100644 index 0000000..67949cc --- /dev/null +++ b/web-application/backend/test/mocha/api-tests/tests-api-personas.ts @@ -0,0 +1,84 @@ +// API tests + +"use strict"; + +import assert from "assert"; +import type { RequestParams } from "@asanrom/request-axios"; +import { APITester } from "../test-tools/api-tester"; +import { APIAuthentication } from "../test-tools/authentication"; +import { getApiUrl } from "../test-tools/api-bindings/utils"; + +interface PersonaSummary { + id: string; + name: string; + era: string; + nationality: string; + profession: string; + avatar: string; + firstMessage: string; +} + +interface PersonaEmotionalTrigger { + emotion: string; + trigger: string; +} + +interface PersonaDetail extends PersonaSummary { + emotionalProfile: PersonaEmotionalTrigger[]; + searchKeywords: string[]; +} + +function listPersonasRequest(): RequestParams { + return { + method: "GET", + url: getApiUrl("/personas"), + }; +} + +function getPersonaRequest(id: string): RequestParams { + return { + method: "GET", + url: getApiUrl("/personas/" + encodeURIComponent(id)), + }; +} + +describe("API / Personas", () => { + before(async () => { + await APITester.Initialize(); + }); + + it("Should list available personas", async () => { + const personas = await APITester.Test(listPersonasRequest(), APIAuthentication.Unauthenticated()); + + assert.ok(Array.isArray(personas)); + assert.ok(personas.length > 0); + + const first = personas[0]; + assert.ok(first.id); + assert.ok(first.name); + assert.ok(first.era); + assert.ok(first.profession); + }); + + it("Should get a persona by id", async () => { + const personas = await APITester.Test(listPersonasRequest(), APIAuthentication.Unauthenticated()); + const id = personas[0].id; + + const persona = await APITester.Test(getPersonaRequest(id), APIAuthentication.Unauthenticated()); + + assert.equal(persona.id, id); + assert.ok(Array.isArray(persona.emotionalProfile)); + assert.ok(Array.isArray(persona.searchKeywords)); + }); + + it("Should return PERSONA_NOT_FOUND for unknown persona id", async () => { + const unknownId = "persona-does-not-exist-" + Date.now(); + await APITester.TestError( + getPersonaRequest(unknownId), + APIAuthentication.Unauthenticated(), + 404, + "PERSONA_NOT_FOUND", + ); + }); +}); + diff --git a/web-application/backend/test/mocha/api-tests/tests-api-wallet.ts b/web-application/backend/test/mocha/api-tests/tests-api-wallet.ts deleted file mode 100644 index cbd7ca3..0000000 --- a/web-application/backend/test/mocha/api-tests/tests-api-wallet.ts +++ /dev/null @@ -1,96 +0,0 @@ -// API tests - -"use strict"; - -import assert from "assert"; -import Crypto from "crypto"; -import { APITester } from '../test-tools/api-tester'; -import { PreparedUser, TestUsers } from '../test-tools/models/users'; -import { ApiWallet } from '../test-tools/api-bindings/api-group-wallet'; -import { privateKeyToAddress } from '@asanrom/smart-contract-wrapper'; - - -// Test group -describe("API / Wallets", () => { - let testUser1: PreparedUser; - - before(async () => { - // Setup test server (REQUIRED) - await APITester.Initialize(); - - // Setup users - testUser1 = await TestUsers.NewRegularUser(); - }); - - // Tests - - const randomPassword = Crypto.randomBytes(8).toString("hex"); - let wallet1: string; - let wallet2: string; - - it('Should be able to create a wallet', async () => { - const r = await APITester.Test(ApiWallet.CreateWallet({ name: "Wallet 1", password: randomPassword }), testUser1.auth); - - assert.equal(r.name, "Wallet 1"); - - wallet1 = r.id; - }); - - const randomPrivateKey = Crypto.randomBytes(32); - const randomPrivateKeyAddress = privateKeyToAddress(randomPrivateKey); - - it('Should be able to create a wallet with a private key', async () => { - const r = await APITester.Test(ApiWallet.CreateWallet({ name: "Wallet 2", password: randomPassword, private_key: randomPrivateKey.toString("hex") }), testUser1.auth); - - assert.equal(r.name, "Wallet 2"); - assert.equal((r.address || "").toLowerCase(), randomPrivateKeyAddress.toLowerCase()); - - wallet2 = r.id; - }); - - it('Should be able to list the wallets', async () => { - const wallets = await APITester.Test(ApiWallet.ListWallets(), testUser1.auth); - - assert.equal(wallets.length, 2); - }); - - it('Should be able to change the wallet name', async () => { - await APITester.Test(ApiWallet.ModifyWallet(wallet1, { name: "Wallet 1 (M)" }), testUser1.auth); - - const wallet = await APITester.Test(ApiWallet.GetWallet(wallet1), testUser1.auth); - - assert.equal(wallet.name, "Wallet 1 (M)"); - }); - - const randomPassword2 = Crypto.randomBytes(8).toString("hex"); - - it('Should be able to change the wallet password', async () => { - // Should fail if the password is wrong - await APITester.TestError(ApiWallet.ChangeWalletPassword(wallet1, {password: "Invalid", new_password: randomPassword2}), testUser1.auth, 400, "WRONG_PASSWORD"); - - // Success if correct password - await APITester.Test(ApiWallet.ChangeWalletPassword(wallet1, {password: randomPassword, new_password: randomPassword2}), testUser1.auth) - }); - - it('Should be able to export the private key', async () => { - // Should fail if wrong password - await APITester.TestError(ApiWallet.ExportPrivatekey(wallet1, {password: "Invalid"}), testUser1.auth, 400, "WRONG_PASSWORD"); - - // Success if correct password - await APITester.Test(ApiWallet.ExportPrivatekey(wallet1, {password: randomPassword2}), testUser1.auth); - - const r = await APITester.Test(ApiWallet.ExportPrivatekey(wallet2, {password: randomPassword}), testUser1.auth); - - assert.equal(r.private_key.toUpperCase(), randomPrivateKey.toString("hex").toUpperCase()); - }); - - it('Should be able to delete a wallet', async () => { - await APITester.Test(ApiWallet.DeleteWallet(wallet1), testUser1.auth); - - await APITester.TestError(ApiWallet.GetWallet(wallet1), testUser1.auth, 404); - - const wallets = await APITester.Test(ApiWallet.ListWallets(), testUser1.auth); - - assert.equal(wallets.length, 1); - }); -}); diff --git a/web-application/backend/test/mocha/test-tools/api-bindings/api-group-block-explorer.ts b/web-application/backend/test/mocha/test-tools/api-bindings/api-group-block-explorer.ts deleted file mode 100644 index 60048c4..0000000 --- a/web-application/backend/test/mocha/test-tools/api-bindings/api-group-block-explorer.ts +++ /dev/null @@ -1,289 +0,0 @@ -// API bindings: block_explorer (Auto generated) - -"use strict"; - -import { RequestErrorHandler } from "@asanrom/request-axios"; -import type { RequestParams, CommonAuthenticatedErrorHandler } from "@asanrom/request-axios"; -import { getApiUrl, generateURIQuery } from "./utils"; -import type { ExplorerSearchInformation, BlockInformationMin, GetBlocks, BlockInformation, AccountInformation, TransactionInformation } from "./definitions"; - -export class ApiBlockExplorer { - /** - * Method: GET - * Path: /block-explorer/search - * Searchs block, transaction or account information - * @param queryParams Query parameters - * @returns The request parameters - */ - public static Search(queryParams: SearchQueryParameters): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/search` + generateURIQuery(queryParams)), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "BLOCK_NOT_FOUND", handler.notFoundBlockNotFound) - .add(404, "TRANSACTION_NOT_FOUND", handler.notFoundTransactionNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_BLOCK", handler.badRequestInvalidBlock) - .add(400, "INVALID_ADDRESS", handler.badRequestInvalidAddress) - .add(400, "INVALID_QUERY", handler.badRequestInvalidQuery) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/last-blocks - * Gets last blocks - * @returns The request parameters - */ - public static GetLastBlock(): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/last-blocks`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/blocks - * Gets blocks - * @param queryParams Query parameters - * @returns The request parameters - */ - public static GetBlocks(queryParams: GetBlocksQueryParameters): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/blocks` + generateURIQuery(queryParams)), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/blocks/{block} - * Gets block information - * @param block Block number or hash - * @returns The request parameters - */ - public static GetBlock(block: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/blocks/${encodeURIComponent(block)}`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "BLOCK_NOT_FOUND", handler.notFoundBlockNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_BLOCK", handler.badRequestInvalidBlock) - .add(400, "INVALID_BLOCK_HASH", handler.badRequestInvalidBlockHash) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/account/{address} - * Gets account information - * @param address Account address - * @returns The request parameters - */ - public static GetAccount(address: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/account/${encodeURIComponent(address)}`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(400, "INVALID_WALLET_ADDRESS", handler.badRequestInvalidWalletAddress) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /block-explorer/transactions/{tx} - * Gets transaction information - * @param tx Transaction hash - * @returns The request parameters - */ - public static GetTransaction(tx: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/block-explorer/transactions/${encodeURIComponent(tx)}`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "TRANSACTION_NOT_FOUND", handler.notFoundTransactionNotFound) - .add(404, "*", handler.notFound) - .add(400, "INVALID_TRANSACTION", handler.badRequestInvalidTransaction) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } -} - -/** - * Error handler for Search - */ -export type SearchErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_QUERY - */ - badRequestInvalidQuery: () => void; - - /** - * Handler for status = 400 and code = INVALID_ADDRESS - */ - badRequestInvalidAddress: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK - */ - badRequestInvalidBlock: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = TRANSACTION_NOT_FOUND - */ - notFoundTransactionNotFound: () => void; - - /** - * Handler for status = 404 and code = BLOCK_NOT_FOUND - */ - notFoundBlockNotFound: () => void; -}; - -/** - * Query parameters for Search - */ -export interface SearchQueryParameters { - /** - * Search query - */ - q: string; -} - -/** - * Query parameters for GetBlocks - */ -export interface GetBlocksQueryParameters { - /** - * Continuation block, to request more pages - */ - continuationBlock?: number; - - /** - * Max number of items to fetch per page. Default 25. Max 100. - */ - limit?: number; -} - -/** - * Error handler for GetBlock - */ -export type GetBlockErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK_HASH - */ - badRequestInvalidBlockHash: () => void; - - /** - * Handler for status = 400 and code = INVALID_BLOCK - */ - badRequestInvalidBlock: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = BLOCK_NOT_FOUND - */ - notFoundBlockNotFound: () => void; -}; - -/** - * Error handler for GetAccount - */ -export type GetAccountErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_WALLET_ADDRESS - */ - badRequestInvalidWalletAddress: () => void; -}; - -/** - * Error handler for GetTransaction - */ -export type GetTransactionErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Handler for status = 400 and code = INVALID_TRANSACTION - */ - badRequestInvalidTransaction: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; - - /** - * Handler for status = 404 and code = TRANSACTION_NOT_FOUND - */ - notFoundTransactionNotFound: () => void; -}; - diff --git a/web-application/backend/test/mocha/test-tools/api-bindings/api-group-personas.ts b/web-application/backend/test/mocha/test-tools/api-bindings/api-group-personas.ts new file mode 100644 index 0000000..459f69c --- /dev/null +++ b/web-application/backend/test/mocha/test-tools/api-bindings/api-group-personas.ts @@ -0,0 +1,67 @@ +// API bindings: personas (Auto generated) + +"use strict"; + +import { RequestErrorHandler } from "@asanrom/request-axios"; +import type { RequestParams, CommonErrorHandler } from "@asanrom/request-axios"; +import { getApiUrl } from "./utils"; +import type { PersonaSummary, PersonaDetail } from "./definitions"; + +export class ApiPersonas { + /** + * Method: GET + * Path: /personas + * Lists all available personas + * @returns The request parameters + */ + public static ListPersonas(): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas`), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } + + /** + * Method: GET + * Path: /personas/{id} + * Gets a specific persona by ID + * @param id Persona slug (e.g., "tesla") + * @returns The request parameters + */ + public static GetPersona(id: string): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas/${encodeURIComponent(id)}`), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(404, "PERSONA_NOT_FOUND", handler.notFoundPersonaNotFound) + .add(404, "*", handler.notFound) + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } +} + +/** + * Error handler for GetPersona + */ +export type GetPersonaErrorHandler = CommonErrorHandler & { + /** + * General handler for status = 404 + */ + notFound: () => void; + + /** + * Persona was not found + */ + notFoundPersonaNotFound: () => void; +}; + diff --git a/web-application/backend/test/mocha/test-tools/api-bindings/api-group-wallet.ts b/web-application/backend/test/mocha/test-tools/api-bindings/api-group-wallet.ts deleted file mode 100644 index 68dd926..0000000 --- a/web-application/backend/test/mocha/test-tools/api-bindings/api-group-wallet.ts +++ /dev/null @@ -1,285 +0,0 @@ -// API bindings: wallet (Auto generated) - -"use strict"; - -import { RequestErrorHandler } from "@asanrom/request-axios"; -import type { RequestParams, CommonAuthenticatedErrorHandler } from "@asanrom/request-axios"; -import { getApiUrl } from "./utils"; -import type { WalletInfo, WalletCreateBody, WalletEditBody, WalletChangePasswordBody, WalletExportResponse, WalletExportBody } from "./definitions"; - -export class ApiWallet { - /** - * Method: GET - * Path: /wallet - * List wallets - * @returns The request parameters - */ - public static ListWallets(): RequestParams { - return { - method: "GET", - url: getApiUrl(`/wallet`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: POST - * Path: /wallet - * Create wallet - * @param body Body parameters - * @returns The request parameters - */ - public static CreateWallet(body: WalletCreateBody): RequestParams { - return { - method: "POST", - url: getApiUrl(`/wallet`), - json: body, - handleError: (err, handler) => { - new RequestErrorHandler() - .add(400, "INVALID_PRIVATE_KEY", handler.badRequestInvalidPrivateKey) - .add(400, "WEAK_PASSWORD", handler.badRequestWeakPassword) - .add(400, "TOO_MANY_WALLETS", handler.badRequestTooManyWallets) - .add(400, "INVALID_NAME", handler.badRequestInvalidName) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: GET - * Path: /wallet/{id} - * Get wallet - * @param id Wallet ID - * @returns The request parameters - */ - public static GetWallet(id: string): RequestParams { - return { - method: "GET", - url: getApiUrl(`/wallet/${encodeURIComponent(id)}`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "*", handler.notFound) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: POST - * Path: /wallet/{id} - * Edit wallet - * @param id Wallet ID - * @param body Body parameters - * @returns The request parameters - */ - public static ModifyWallet(id: string, body: WalletEditBody): RequestParams { - return { - method: "POST", - url: getApiUrl(`/wallet/${encodeURIComponent(id)}`), - json: body, - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "*", handler.notFound) - .add(400, "INVALID_NAME", handler.badRequestInvalidName) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: DELETE - * Path: /wallet/{id} - * Deletes a wallet - * @param id Wallet ID - * @returns The request parameters - */ - public static DeleteWallet(id: string): RequestParams { - return { - method: "DELETE", - url: getApiUrl(`/wallet/${encodeURIComponent(id)}`), - handleError: (err, handler) => { - new RequestErrorHandler() - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: POST - * Path: /wallet/{id}/password - * Change wallet password - * @param id Wallet ID - * @param body Body parameters - * @returns The request parameters - */ - public static ChangeWalletPassword(id: string, body: WalletChangePasswordBody): RequestParams { - return { - method: "POST", - url: getApiUrl(`/wallet/${encodeURIComponent(id)}/password`), - json: body, - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "*", handler.notFound) - .add(400, "WRONG_PASSWORD", handler.badRequestWrongPassword) - .add(400, "WEAK_PASSWORD", handler.badRequestWeakPassword) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } - - /** - * Method: POST - * Path: /wallet/{id}/export - * Export wallet private key - * @param id Wallet ID - * @param body Body parameters - * @returns The request parameters - */ - public static ExportPrivatekey(id: string, body: WalletExportBody): RequestParams { - return { - method: "POST", - url: getApiUrl(`/wallet/${encodeURIComponent(id)}/export`), - json: body, - handleError: (err, handler) => { - new RequestErrorHandler() - .add(404, "*", handler.notFound) - .add(400, "WRONG_PASSWORD", handler.badRequestWrongPassword) - .add(400, "*", handler.badRequest) - .add(401, "*", handler.unauthorized) - .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) - .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) - .handle(err); - }, - }; - } -} - -/** - * Error handler for CreateWallet - */ -export type CreateWalletErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Invalid wallet name - */ - badRequestInvalidName: () => void; - - /** - * You have too many wallets - */ - badRequestTooManyWallets: () => void; - - /** - * Password too weak - */ - badRequestWeakPassword: () => void; - - /** - * Invalid private key provided - */ - badRequestInvalidPrivateKey: () => void; -}; - -/** - * Error handler for GetWallet - */ -export type GetWalletErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 404 - */ - notFound: () => void; -}; - -/** - * Error handler for ModifyWallet - */ -export type ModifyWalletErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Invalid wallet name - */ - badRequestInvalidName: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; -}; - -/** - * Error handler for ChangeWalletPassword - */ -export type ChangeWalletPasswordErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Password too weak - */ - badRequestWeakPassword: () => void; - - /** - * Wrong current password - */ - badRequestWrongPassword: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; -}; - -/** - * Error handler for ExportPrivatekey - */ -export type ExportPrivatekeyErrorHandler = CommonAuthenticatedErrorHandler & { - /** - * General handler for status = 400 - */ - badRequest: () => void; - - /** - * Wrong current password - */ - badRequestWrongPassword: () => void; - - /** - * General handler for status = 404 - */ - notFound: () => void; -}; - diff --git a/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts b/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts index 5e307f0..b9b85ff 100644 --- a/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts +++ b/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts @@ -2,109 +2,6 @@ "use strict" -export interface WalletInfo { - /** - * Wallet ID - */ - id: string; - - /** - * Wallet name - */ - name?: string; - - /** - * Wallet address - */ - address?: string; -} - -export interface WalletCreateBody { - /** - * Wallet name (max 80 chars) - */ - name: string; - - /** - * Password to protect the wallet - */ - password: string; - - /** - * Private key (HEX). If not provided, a random one is generated. - */ - private_key?: string; -} - -export interface WalletCreateBadRequest { - /** - * Error Code: - * - INVALID_NAME: Invalid wallet name - * - TOO_MANY_WALLETS: You have too many wallets - * - WEAK_PASSWORD: Password too weak - * - INVALID_PRIVATE_KEY: Invalid private key provided - */ - code: string; -} - -export interface WalletEditBody { - /** - * Wallet name (max 80 chars) - */ - name: string; -} - -export interface WalletEditBadRequest { - /** - * Error Code: - * - INVALID_NAME: Invalid wallet name - */ - code: string; -} - -export interface WalletChangePasswordBody { - /** - * Current password - */ - password: string; - - /** - * New password - */ - new_password: string; -} - -export interface WalletChangePasswordBadRequest { - /** - * Error Code: - * - WEAK_PASSWORD: Password too weak - * - WRONG_PASSWORD: Wrong current password - */ - code: string; -} - -export interface WalletExportBody { - /** - * Current password - */ - password: string; -} - -export interface WalletExportBadRequest { - /** - * Error Code: - * - WRONG_PASSWORD: Wrong current password - */ - code: string; -} - -export interface WalletExportResponse { - /** - * Private key (HEX) - */ - private_key: string; -} - export interface UserAdminListItem { /** * User ID @@ -416,466 +313,102 @@ export interface UploadProfileImageResponse { url?: string; } -export interface ExplorerSearchInformationItem { - /** - * Account balance (ETH) - */ - balance?: number; - - /** - * Total transactions - */ - totalTransactions?: number; - - /** - * True if address is a contract, false if not - */ - isContract?: boolean; - - /** - * Block number - */ - number?: number; - - /** - * Mix hash - */ - mixHash?: string; - - /** - * Parent hash - */ - parentHash?: string; - - /** - * sha3Uncles - */ - sha3Uncles?: string; - - /** - * Logs bloom - */ - logsBloom?: string; - - /** - * Transactions root - */ - transactionsRoot?: string; - - /** - * State root - */ - stateRoot?: string; - - /** - * Receipts root - */ - receiptsRoot?: string; - - /** - * Miner address - */ - miner?: string; - - /** - * Difficulty - */ - difficulty?: number; - - /** - * Total difficulty - */ - totalDifficulty?: number; - - /** - * Extra data - */ - extraData?: string; - - /** - * Block size - */ - size?: number; - - /** - * Gas limit - */ - gasLimit?: number; - - /** - * Gas used - */ - gasUsed?: number; - - /** - * Base fee per gas - */ - baseFeePerGas?: number; - - /** - * Block timestamp - */ - timestamp?: number; - - transactions?: string[]; - - /** - * Block hash - */ - blockHash?: string; - - /** - * Block number - */ - blockNumber?: number; - - /** - * Chain ID - */ - chainId?: string; - - /** - * From address - */ - from?: string; - - /** - * Gas - */ - gas?: number; - - /** - * Gas price - */ - gasPrice?: number; - - /** - * Max priority fee per gas - */ - maxPriorityFeePerGas?: number; - - /** - * Max fee per gas - */ - maxFeePerGas?: number; - - /** - * Block or transaction hash - */ - hash?: string; - - /** - * Transaction input - */ - input?: string; - - /** - * Nonce - */ - nonce?: string; - +export interface PersonaSummary { /** - * To address + * Persona slug identifier */ - to?: string; - - /** - * Transaction index - */ - transactionIndex?: number; - - /** - * Transaction type - */ - type?: number; + id?: string; /** - * Transaction value + * Display name */ - value?: number; + name?: string; /** - * y parity + * Birth-death years */ - yParity?: number; + era?: string; /** - * V + * Nationality */ - v?: string; + nationality?: string; /** - * R + * Profession / title */ - r?: string; + profession?: string; /** - * S + * Avatar image path */ - s?: string; -} + avatar?: string; -export interface ExplorerSearchInformation { /** - * Mode: account, block, transaction + * Greeting message */ - mode: string; - - info?: ExplorerSearchInformationItem; + firstMessage?: string; } -export interface BlockInformationMin { - /** - * Block number - */ - number?: number; - - /** - * Block hash - */ - hash?: string; - - /** - * Miner address - */ - miner?: string; - +export interface PersonaEmotionalTrigger { /** - * Block size + * Emotional mode (e.g. "angry") */ - size?: number; + emotion?: string; /** - * Timestamp + * Trigger context for that emotion */ - timestamp?: number; - - /** - * Transactions length - */ - transactions?: number; -} - -export interface GetBlocks { - blocks: BlockInformationMin[]; - - /** - * Continuation block number - */ - continuationBlock?: number; + trigger?: string; } -export interface BlockInformation { +export interface PersonaDetail { /** - * Block number + * Persona slug identifier */ - number?: number; - - /** - * Block hash - */ - hash?: string; - - /** - * Mix hash - */ - mixHash?: string; - - /** - * Parent hash - */ - parentHash?: string; - - /** - * Nonce - */ - nonce?: string; - - /** - * sha3Uncles - */ - sha3Uncles?: string; - - /** - * Logs bloom - */ - logsBloom?: string; - - /** - * Transactions root - */ - transactionsRoot?: string; - - /** - * State root - */ - stateRoot?: string; - - /** - * Receipts root - */ - receiptsRoot?: string; - - /** - * Miner address - */ - miner?: string; - - /** - * Difficulty - */ - difficulty?: number; - - /** - * Total difficulty - */ - totalDifficulty?: number; - - /** - * Extra data - */ - extraData?: string; + id?: string; /** - * Block size + * Display name */ - size?: number; + name?: string; /** - * Gas limit + * Birth-death years */ - gasLimit?: number; + era?: string; /** - * Gas used + * Nationality */ - gasUsed?: number; + nationality?: string; /** - * Base fee per gas + * Profession / title */ - baseFeePerGas?: number; + profession?: string; /** - * Block timestamp + * Avatar image path */ - timestamp?: number; + avatar?: string; - transactions?: string[]; -} - -export interface AccountInformation { /** - * Account balance + * Greeting message */ - balance?: number; + firstMessage?: string; - /** - * Total transactions - */ - totalTransactions?: number; + emotionalProfile?: PersonaEmotionalTrigger[]; - /** - * True if address is a contract, false if not - */ - isContract?: boolean; + searchKeywords?: string[]; } -export interface TransactionInformation { - /** - * Block hash - */ - blockHash?: string; - - /** - * Block number - */ - blockNumber?: number; - - /** - * Chain ID - */ - chainId?: string; - - /** - * From address - */ - from?: string; - - /** - * Gas - */ - gas?: number; - - /** - * Gas price - */ - gasPrice?: number; - - /** - * Max priority fee per gas - */ - maxPriorityFeePerGas?: number; - - /** - * Max fee per gas - */ - maxFeePerGas?: number; - +export interface PersonaNotFoundError { /** - * Transaction hash - */ - hash?: string; - - /** - * Transaction input - */ - input?: string; - - /** - * Nonce - */ - nonce?: string; - - /** - * To address - */ - to?: string; - - /** - * Transaction index - */ - transactionIndex?: number; - - /** - * Transaction type - */ - type?: number; - - /** - * Transaction value - */ - value?: number; - - /** - * y parity - */ - yParity?: number; - - /** - * V - */ - v?: string; - - /** - * R - */ - r?: string; - - /** - * S + * Error code: + * - PERSONA_NOT_FOUND: Persona was not found */ - s?: string; + code: string; } export interface ErrorResponse { diff --git a/web-application/frontend/api/api-group-personas.ts b/web-application/frontend/api/api-group-personas.ts new file mode 100644 index 0000000..dce80ab --- /dev/null +++ b/web-application/frontend/api/api-group-personas.ts @@ -0,0 +1,67 @@ +// API bindings: personas (Auto generated) + +"use strict"; + +import { RequestErrorHandler } from "@asanrom/request-browser"; +import type { RequestParams, CommonErrorHandler } from "@asanrom/request-browser"; +import { getApiUrl } from "./utils"; +import type { PersonaSummary, PersonaDetail } from "./definitions"; + +export class ApiPersonas { + /** + * Method: GET + * Path: /personas + * Lists all available personas + * @returns The request parameters + */ + public static ListPersonas(): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas`), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } + + /** + * Method: GET + * Path: /personas/{id} + * Gets a specific persona by ID + * @param id Persona slug (e.g., "tesla") + * @returns The request parameters + */ + public static GetPersona(id: string): RequestParams { + return { + method: "GET", + url: getApiUrl(`/personas/${encodeURIComponent(id)}`), + handleError: (err, handler) => { + new RequestErrorHandler() + .add(404, "PERSONA_NOT_FOUND", handler.notFoundPersonaNotFound) + .add(404, "*", handler.notFound) + .add(500, "*", "serverError" in handler ? handler.serverError : handler.temporalError) + .add("*", "*", "networkError" in handler ? handler.networkError : handler.temporalError) + .handle(err); + }, + }; + } +} + +/** + * Error handler for GetPersona + */ +export type GetPersonaErrorHandler = CommonErrorHandler & { + /** + * General handler for status = 404 + */ + notFound: () => void; + + /** + * Persona was not found + */ + notFoundPersonaNotFound: () => void; +}; + diff --git a/web-application/frontend/api/definitions.ts b/web-application/frontend/api/definitions.ts index 2435ea8..b9b85ff 100644 --- a/web-application/frontend/api/definitions.ts +++ b/web-application/frontend/api/definitions.ts @@ -313,6 +313,104 @@ export interface UploadProfileImageResponse { url?: string; } +export interface PersonaSummary { + /** + * Persona slug identifier + */ + id?: string; + + /** + * Display name + */ + name?: string; + + /** + * Birth-death years + */ + era?: string; + + /** + * Nationality + */ + nationality?: string; + + /** + * Profession / title + */ + profession?: string; + + /** + * Avatar image path + */ + avatar?: string; + + /** + * Greeting message + */ + firstMessage?: string; +} + +export interface PersonaEmotionalTrigger { + /** + * Emotional mode (e.g. "angry") + */ + emotion?: string; + + /** + * Trigger context for that emotion + */ + trigger?: string; +} + +export interface PersonaDetail { + /** + * Persona slug identifier + */ + id?: string; + + /** + * Display name + */ + name?: string; + + /** + * Birth-death years + */ + era?: string; + + /** + * Nationality + */ + nationality?: string; + + /** + * Profession / title + */ + profession?: string; + + /** + * Avatar image path + */ + avatar?: string; + + /** + * Greeting message + */ + firstMessage?: string; + + emotionalProfile?: PersonaEmotionalTrigger[]; + + searchKeywords?: string[]; +} + +export interface PersonaNotFoundError { + /** + * Error code: + * - PERSONA_NOT_FOUND: Persona was not found + */ + code: string; +} + export interface ErrorResponse { /** * Error code diff --git a/web-application/frontend/composables/useAudioPlayback.ts b/web-application/frontend/composables/useAudioPlayback.ts index c140862..11a6bbe 100644 --- a/web-application/frontend/composables/useAudioPlayback.ts +++ b/web-application/frontend/composables/useAudioPlayback.ts @@ -7,6 +7,9 @@ export function useAudioPlayback() { const isPlaying = ref(false); let audioContext: AudioContext | null = null; let currentSource: AudioBufferSourceNode | null = null; + let playbackGeneration = 0; + let processingQueue = false; + const playbackQueue: Array<{ chunk: string; onEnded?: () => void }> = []; function getAudioContext(): AudioContext { if (!audioContext || audioContext.state === "closed") { @@ -21,46 +24,86 @@ export function useAudioPlayback() { /** * Plays a base64-encoded audio chunk (MP3/WAV). - * Stops any currently playing audio first. + * Chunks are queued and played sequentially. * @param base64Chunk Base64-encoded audio data * @param onEnded Optional callback when playback finishes */ async function playChunk(base64Chunk: string, onEnded?: () => void) { if (!base64Chunk) return; + playbackQueue.push({ chunk: base64Chunk, onEnded }); + if (processingQueue) { + return; + } + await processQueue(); + } + + async function processQueue() { + if (processingQueue) { + return; + } + processingQueue = true; + try { - const ctx = getAudioContext(); + while (playbackQueue.length > 0) { + const item = playbackQueue.shift(); + if (!item || !item.chunk) { + continue; + } - // Decode base64 to ArrayBuffer - const binaryStr = atob(base64Chunk); - const bytes = new Uint8Array(binaryStr.length); - for (let i = 0; i < binaryStr.length; i++) { - bytes[i] = binaryStr.charCodeAt(i); - } + try { + const generationAtStart = playbackGeneration; + const ctx = getAudioContext(); - // Decode audio data - const audioBuffer = await ctx.decodeAudioData(bytes.buffer.slice(0)); + // Decode base64 to ArrayBuffer + const binaryStr = atob(item.chunk); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } - // Stop previous playback - stopPlayback(); + // Decode audio data + const audioBuffer = await ctx.decodeAudioData(bytes.buffer.slice(0)); - // Create and play source - const source = ctx.createBufferSource(); - source.buffer = audioBuffer; - source.connect(ctx.destination); + // Playback was reset while decoding + if (generationAtStart !== playbackGeneration) { + continue; + } - source.onended = () => { - isPlaying.value = false; - currentSource = null; - if (onEnded) onEnded(); - }; + await new Promise((resolve) => { + const source = ctx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(ctx.destination); - currentSource = source; - isPlaying.value = true; - source.start(0); + source.onended = () => { + if (currentSource === source) { + currentSource = null; + } + if (generationAtStart === playbackGeneration && item.onEnded) { + item.onEnded(); + } + resolve(); + }; + + currentSource = source; + isPlaying.value = true; + source.start(0); + }); + } catch (chunkErr) { + console.warn("useAudioPlayback: failed to decode queued chunk", chunkErr); + } + } } catch (err) { console.warn("useAudioPlayback: failed to play chunk", err); - isPlaying.value = false; + } finally { + processingQueue = false; + if (playbackQueue.length > 0) { + void processQueue(); + return; + } + if (!currentSource && playbackQueue.length === 0) { + isPlaying.value = false; + } } } @@ -68,6 +111,9 @@ export function useAudioPlayback() { * Stops current audio playback. */ function stopPlayback() { + playbackGeneration++; + playbackQueue.length = 0; + if (currentSource) { try { currentSource.stop(); diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts index 64a26bd..863789b 100644 --- a/web-application/frontend/composables/useVoiceInput.ts +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -35,7 +35,12 @@ export function useVoiceInput(sendMessage: (msg: any) => void) { // Dynamically import VAD to avoid SSR issues const { MicVAD } = await import("@ricky0123/vad-web"); + const vadAssetBase = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/"; + const onnxWasmBase = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.24.3/dist/"; + vadInstance = await MicVAD.new({ + baseAssetPath: vadAssetBase, + onnxWASMBasePath: onnxWasmBase, getStream: async () => mediaStream!, onSpeechStart: () => { isSpeaking.value = true; @@ -153,9 +158,13 @@ function writeString(view: DataView, offset: number, str: string) { function arrayBufferToBase64(buffer: ArrayBuffer): string { const bytes = new Uint8Array(buffer); - let binary = ""; - for (let i = 0; i < bytes.length; i++) { - binary += String.fromCharCode(bytes[i]); + const chunkSize = 0x8000; + const binaryChunks: string[] = []; + + for (let i = 0; i < bytes.length; i += chunkSize) { + const chunk = bytes.subarray(i, i + chunkSize); + binaryChunks.push(String.fromCharCode.apply(null, Array.from(chunk))); } - return btoa(binary); + + return btoa(binaryChunks.join("")); } diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index cdb7edb..4fbb06c 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -61,6 +61,8 @@ diff --git a/web-application/frontend/components/source-cards/SourceCardItem.vue b/web-application/frontend/components/source-cards/SourceCardItem.vue index 2eac32e..47580e1 100644 --- a/web-application/frontend/components/source-cards/SourceCardItem.vue +++ b/web-application/frontend/components/source-cards/SourceCardItem.vue @@ -1,34 +1,38 @@ diff --git a/web-application/frontend/composables/useAudioPlayback.ts b/web-application/frontend/composables/useAudioPlayback.ts index 11a6bbe..04e9b7b 100644 --- a/web-application/frontend/composables/useAudioPlayback.ts +++ b/web-application/frontend/composables/useAudioPlayback.ts @@ -4,139 +4,139 @@ */ export function useAudioPlayback() { - const isPlaying = ref(false); - let audioContext: AudioContext | null = null; - let currentSource: AudioBufferSourceNode | null = null; - let playbackGeneration = 0; - let processingQueue = false; - const playbackQueue: Array<{ chunk: string; onEnded?: () => void }> = []; - - function getAudioContext(): AudioContext { - if (!audioContext || audioContext.state === "closed") { - audioContext = new AudioContext(); - } - // Resume if suspended (browser autoplay policy) - if (audioContext.state === "suspended") { - audioContext.resume(); - } - return audioContext; - } - - /** - * Plays a base64-encoded audio chunk (MP3/WAV). - * Chunks are queued and played sequentially. - * @param base64Chunk Base64-encoded audio data - * @param onEnded Optional callback when playback finishes - */ - async function playChunk(base64Chunk: string, onEnded?: () => void) { - if (!base64Chunk) return; - - playbackQueue.push({ chunk: base64Chunk, onEnded }); - if (processingQueue) { - return; + const isPlaying = ref(false); + let audioContext: AudioContext | null = null; + let currentSource: AudioBufferSourceNode | null = null; + let playbackGeneration = 0; + let processingQueue = false; + const playbackQueue: Array<{ chunk: string; onEnded?: () => void }> = []; + + function getAudioContext(): AudioContext { + if (!audioContext || audioContext.state === "closed") { + audioContext = new AudioContext(); + } + // Resume if suspended (browser autoplay policy) + if (audioContext.state === "suspended") { + audioContext.resume(); + } + return audioContext; } - await processQueue(); - } - async function processQueue() { - if (processingQueue) { - return; + /** + * Plays a base64-encoded audio chunk (MP3/WAV). + * Chunks are queued and played sequentially. + * @param base64Chunk Base64-encoded audio data + * @param onEnded Optional callback when playback finishes + */ + async function playChunk(base64Chunk: string, onEnded?: () => void) { + if (!base64Chunk) return; + + playbackQueue.push({ chunk: base64Chunk, onEnded }); + if (processingQueue) { + return; + } + await processQueue(); } - processingQueue = true; - try { - while (playbackQueue.length > 0) { - const item = playbackQueue.shift(); - if (!item || !item.chunk) { - continue; + async function processQueue() { + if (processingQueue) { + return; } + processingQueue = true; try { - const generationAtStart = playbackGeneration; - const ctx = getAudioContext(); - - // Decode base64 to ArrayBuffer - const binaryStr = atob(item.chunk); - const bytes = new Uint8Array(binaryStr.length); - for (let i = 0; i < binaryStr.length; i++) { - bytes[i] = binaryStr.charCodeAt(i); - } - - // Decode audio data - const audioBuffer = await ctx.decodeAudioData(bytes.buffer.slice(0)); - - // Playback was reset while decoding - if (generationAtStart !== playbackGeneration) { - continue; - } - - await new Promise((resolve) => { - const source = ctx.createBufferSource(); - source.buffer = audioBuffer; - source.connect(ctx.destination); - - source.onended = () => { - if (currentSource === source) { - currentSource = null; - } - if (generationAtStart === playbackGeneration && item.onEnded) { - item.onEnded(); - } - resolve(); - }; - - currentSource = source; - isPlaying.value = true; - source.start(0); - }); - } catch (chunkErr) { - console.warn("useAudioPlayback: failed to decode queued chunk", chunkErr); + while (playbackQueue.length > 0) { + const item = playbackQueue.shift(); + if (!item || !item.chunk) { + continue; + } + + try { + const generationAtStart = playbackGeneration; + const ctx = getAudioContext(); + + // Decode base64 to ArrayBuffer + const binaryStr = atob(item.chunk); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + + // Decode audio data + const audioBuffer = await ctx.decodeAudioData(bytes.buffer.slice(0)); + + // Playback was reset while decoding + if (generationAtStart !== playbackGeneration) { + continue; + } + + await new Promise((resolve) => { + const source = ctx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(ctx.destination); + + source.onended = () => { + if (currentSource === source) { + currentSource = null; + } + if (generationAtStart === playbackGeneration && item.onEnded) { + item.onEnded(); + } + resolve(); + }; + + currentSource = source; + isPlaying.value = true; + source.start(0); + }); + } catch (chunkErr) { + console.warn("useAudioPlayback: failed to decode queued chunk", chunkErr); + } + } + } catch (err) { + console.warn("useAudioPlayback: failed to play chunk", err); + } finally { + processingQueue = false; + if (playbackQueue.length > 0) { + void processQueue(); + return; + } + if (!currentSource && playbackQueue.length === 0) { + isPlaying.value = false; + } } - } - } catch (err) { - console.warn("useAudioPlayback: failed to play chunk", err); - } finally { - processingQueue = false; - if (playbackQueue.length > 0) { - void processQueue(); - return; - } - if (!currentSource && playbackQueue.length === 0) { - isPlaying.value = false; - } } - } - - /** - * Stops current audio playback. - */ - function stopPlayback() { - playbackGeneration++; - playbackQueue.length = 0; - - if (currentSource) { - try { - currentSource.stop(); - } catch (_e) { - // Ignore if already stopped - } - currentSource = null; - } - isPlaying.value = false; - } - - // Cleanup on unmount - onBeforeUnmount(() => { - stopPlayback(); - if (audioContext && audioContext.state !== "closed") { - audioContext.close(); - audioContext = null; + + /** + * Stops current audio playback. + */ + function stopPlayback() { + playbackGeneration++; + playbackQueue.length = 0; + + if (currentSource) { + try { + currentSource.stop(); + } catch (_e) { + // Ignore if already stopped + } + currentSource = null; + } + isPlaying.value = false; } - }); - return { - isPlaying, - playChunk, - stopPlayback, - }; + // Cleanup on unmount + onBeforeUnmount(() => { + stopPlayback(); + if (audioContext && audioContext.state !== "closed") { + audioContext.close(); + audioContext = null; + } + }); + + return { + isPlaying, + playChunk, + stopPlayback, + }; } diff --git a/web-application/frontend/composables/useOrchestratorSocket.ts b/web-application/frontend/composables/useOrchestratorSocket.ts index 57c724c..ce07ef9 100644 --- a/web-application/frontend/composables/useOrchestratorSocket.ts +++ b/web-application/frontend/composables/useOrchestratorSocket.ts @@ -27,7 +27,13 @@ export function useOrchestratorSocket() { }); // VueUse WebSocket (auto-imported by @vueuse/nuxt) - const { status, data, send: wsSend, open, close } = useWebSocket(wsUrl, { + const { + status, + data, + send: wsSend, + open, + close, + } = useWebSocket(wsUrl, { autoReconnect: { retries: 5, delay: 2000, diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts index 863789b..3fcd64d 100644 --- a/web-application/frontend/composables/useVoiceInput.ts +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -5,166 +5,166 @@ */ export function useVoiceInput(sendMessage: (msg: any) => void) { - const isListening = ref(false); - const isSpeaking = ref(false); - const error = ref(null); - - let mediaStream: MediaStream | null = null; - let vadInstance: any = null; - - /** - * Starts listening for voice input. - * Requests microphone access and initializes VAD. - */ - async function startListening() { - if (isListening.value) return; - error.value = null; - - try { - // Request microphone access - mediaStream = await navigator.mediaDevices.getUserMedia({ - audio: { - sampleRate: 16000, - channelCount: 1, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }); - - // Dynamically import VAD to avoid SSR issues - const { MicVAD } = await import("@ricky0123/vad-web"); - - const vadAssetBase = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/"; - const onnxWasmBase = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.24.3/dist/"; - - vadInstance = await MicVAD.new({ - baseAssetPath: vadAssetBase, - onnxWASMBasePath: onnxWasmBase, - getStream: async () => mediaStream!, - onSpeechStart: () => { - isSpeaking.value = true; - }, - onSpeechEnd: (audio: Float32Array) => { - isSpeaking.value = false; - - // Convert Float32 PCM to 16-bit PCM WAV and send - const wavBuffer = float32ToWav(audio, 16000); - const base64 = arrayBufferToBase64(wavBuffer); - - // Send as a single chunk + speech-end - sendMessage({ type: "audio-chunk", chunk: base64 }); - sendMessage({ type: "speech-end" }); - }, - positiveSpeechThreshold: 0.8, - negativeSpeechThreshold: 0.4, - redemptionMs: 250, - preSpeechPadMs: 100, - minSpeechMs: 150, - }); - - vadInstance.start(); - isListening.value = true; - } catch (err: any) { - error.value = err.message || "Failed to access microphone"; - stopListening(); - } - } - - /** - * Stops listening and releases resources. - */ - function stopListening() { - if (vadInstance) { - try { - vadInstance.destroy(); - } catch (_e) { - // Ignore cleanup errors - } - vadInstance = null; + const isListening = ref(false); + const isSpeaking = ref(false); + const error = ref(null); + + let mediaStream: MediaStream | null = null; + let vadInstance: any = null; + + /** + * Starts listening for voice input. + * Requests microphone access and initializes VAD. + */ + async function startListening() { + if (isListening.value) return; + error.value = null; + + try { + // Request microphone access + mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: 16000, + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + // Dynamically import VAD to avoid SSR issues + const { MicVAD } = await import("@ricky0123/vad-web"); + + const vadAssetBase = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/"; + const onnxWasmBase = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.24.3/dist/"; + + vadInstance = await MicVAD.new({ + baseAssetPath: vadAssetBase, + onnxWASMBasePath: onnxWasmBase, + getStream: async () => mediaStream!, + onSpeechStart: () => { + isSpeaking.value = true; + }, + onSpeechEnd: (audio: Float32Array) => { + isSpeaking.value = false; + + // Convert Float32 PCM to 16-bit PCM WAV and send + const wavBuffer = float32ToWav(audio, 16000); + const base64 = arrayBufferToBase64(wavBuffer); + + // Send as a single chunk + speech-end + sendMessage({ type: "audio-chunk", chunk: base64 }); + sendMessage({ type: "speech-end" }); + }, + positiveSpeechThreshold: 0.8, + negativeSpeechThreshold: 0.4, + redemptionMs: 250, + preSpeechPadMs: 100, + minSpeechMs: 150, + }); + + vadInstance.start(); + isListening.value = true; + } catch (err: any) { + error.value = err.message || "Failed to access microphone"; + stopListening(); + } } - if (mediaStream) { - mediaStream.getTracks().forEach((track) => track.stop()); - mediaStream = null; + /** + * Stops listening and releases resources. + */ + function stopListening() { + if (vadInstance) { + try { + vadInstance.destroy(); + } catch (_e) { + // Ignore cleanup errors + } + vadInstance = null; + } + + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + mediaStream = null; + } + + isListening.value = false; + isSpeaking.value = false; } - isListening.value = false; - isSpeaking.value = false; - } - - // Cleanup on unmount - onBeforeUnmount(() => { - stopListening(); - }); - - return { - isListening, - isSpeaking, - error, - startListening, - stopListening, - }; + // Cleanup on unmount + onBeforeUnmount(() => { + stopListening(); + }); + + return { + isListening, + isSpeaking, + error, + startListening, + stopListening, + }; } /** * Converts a Float32Array of PCM samples to a WAV file ArrayBuffer. */ function float32ToWav(samples: Float32Array, sampleRate: number): ArrayBuffer { - const numChannels = 1; - const bitsPerSample = 16; - const byteRate = sampleRate * numChannels * (bitsPerSample / 8); - const blockAlign = numChannels * (bitsPerSample / 8); - const dataSize = samples.length * (bitsPerSample / 8); - const headerSize = 44; - const buffer = new ArrayBuffer(headerSize + dataSize); - const view = new DataView(buffer); - - // RIFF header - writeString(view, 0, "RIFF"); - view.setUint32(4, 36 + dataSize, true); - writeString(view, 8, "WAVE"); - - // fmt sub-chunk - writeString(view, 12, "fmt "); - view.setUint32(16, 16, true); // Sub-chunk size - view.setUint16(20, 1, true); // PCM format - view.setUint16(22, numChannels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, byteRate, true); - view.setUint16(32, blockAlign, true); - view.setUint16(34, bitsPerSample, true); - - // data sub-chunk - writeString(view, 36, "data"); - view.setUint32(40, dataSize, true); - - // Write samples as 16-bit PCM - let offset = 44; - for (let i = 0; i < samples.length; i++) { - const s = Math.max(-1, Math.min(1, samples[i])); - view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); - offset += 2; - } - - return buffer; + const numChannels = 1; + const bitsPerSample = 16; + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const blockAlign = numChannels * (bitsPerSample / 8); + const dataSize = samples.length * (bitsPerSample / 8); + const headerSize = 44; + const buffer = new ArrayBuffer(headerSize + dataSize); + const view = new DataView(buffer); + + // RIFF header + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + dataSize, true); + writeString(view, 8, "WAVE"); + + // fmt sub-chunk + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // Sub-chunk size + view.setUint16(20, 1, true); // PCM format + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitsPerSample, true); + + // data sub-chunk + writeString(view, 36, "data"); + view.setUint32(40, dataSize, true); + + // Write samples as 16-bit PCM + let offset = 44; + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + offset += 2; + } + + return buffer; } function writeString(view: DataView, offset: number, str: string) { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); - } + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } } function arrayBufferToBase64(buffer: ArrayBuffer): string { - const bytes = new Uint8Array(buffer); - const chunkSize = 0x8000; - const binaryChunks: string[] = []; + const bytes = new Uint8Array(buffer); + const chunkSize = 0x8000; + const binaryChunks: string[] = []; - for (let i = 0; i < bytes.length; i += chunkSize) { - const chunk = bytes.subarray(i, i + chunkSize); - binaryChunks.push(String.fromCharCode.apply(null, Array.from(chunk))); - } + for (let i = 0; i < bytes.length; i += chunkSize) { + const chunk = bytes.subarray(i, i + chunkSize); + binaryChunks.push(String.fromCharCode.apply(null, Array.from(chunk))); + } - return btoa(binaryChunks.join("")); + return btoa(binaryChunks.join("")); } diff --git a/web-application/frontend/models/session.ts b/web-application/frontend/models/session.ts index 8383d9d..18cf848 100644 --- a/web-application/frontend/models/session.ts +++ b/web-application/frontend/models/session.ts @@ -3,23 +3,23 @@ */ export interface Agent { - id: string; - name: string; - voiceId: string; - avatar: string; - stance: "for" | "against" | ""; - era?: string; - profession?: string; + id: string; + name: string; + voiceId: string; + avatar: string; + stance: "for" | "against" | ""; + era?: string; + profession?: string; } export interface PersonaSummary { - id: string; - name: string; - era: string; - nationality: string; - profession: string; - avatar: string; - firstMessage: string; + id: string; + name: string; + era: string; + nationality: string; + profession: string; + avatar: string; + firstMessage: string; } export type SessionStatus = "idle" | "connecting" | "active" | "ended"; diff --git a/web-application/frontend/models/source-card.ts b/web-application/frontend/models/source-card.ts index ef2c378..afff52e 100644 --- a/web-application/frontend/models/source-card.ts +++ b/web-application/frontend/models/source-card.ts @@ -3,11 +3,11 @@ * Used in both Argue.AI (debate evidence) and DeadTalk (biographical sources). */ export interface SourceCard { - id: string; - title: string; - url: string; - snippet: string; - agentId: string; - timestamp: number; - favicon?: string; + id: string; + title: string; + url: string; + snippet: string; + agentId: string; + timestamp: number; + favicon?: string; } diff --git a/web-application/frontend/models/ws-events.ts b/web-application/frontend/models/ws-events.ts index 19dcb57..cb0c250 100644 --- a/web-application/frontend/models/ws-events.ts +++ b/web-application/frontend/models/ws-events.ts @@ -6,103 +6,103 @@ /* Server → Client events */ export interface WsHelloEvent { - event: "hello"; + event: "hello"; } export interface WsSessionStartedEvent { - event: "session-started"; - sessionId: string; + event: "session-started"; + sessionId: string; } export interface WsAgentSpeakingEvent { - event: "agent-speaking"; - agentId: string; - chunk: string; - transcript: string; - audioTag?: string; + event: "agent-speaking"; + agentId: string; + chunk: string; + transcript: string; + audioTag?: string; } export interface WsSourceCitedEvent { - event: "source-cited"; - agentId: string; - source: { - id: string; - title: string; - url: string; - snippet: string; + event: "source-cited"; agentId: string; - timestamp: number; - favicon?: string; - }; + source: { + id: string; + title: string; + url: string; + snippet: string; + agentId: string; + timestamp: number; + favicon?: string; + }; } export interface WsAgentFinishedEvent { - event: "agent-finished"; - agentId: string; + event: "agent-finished"; + agentId: string; } export interface WsSessionEndEvent { - event: "session-end"; - reason: "debate-end" | "conversation-end" | "stopped" | "error"; + event: "session-end"; + reason: "debate-end" | "conversation-end" | "stopped" | "error"; } export interface WsUserTranscriptEvent { - event: "user-transcript"; - text: string; + event: "user-transcript"; + text: string; } export interface WsAgentErrorEvent { - event: "agent-error"; - message: string; + event: "agent-error"; + message: string; } export type WsServerEvent = - | WsHelloEvent - | WsSessionStartedEvent - | WsAgentSpeakingEvent - | WsSourceCitedEvent - | WsAgentFinishedEvent - | WsSessionEndEvent - | WsUserTranscriptEvent - | WsAgentErrorEvent; + | WsHelloEvent + | WsSessionStartedEvent + | WsAgentSpeakingEvent + | WsSourceCitedEvent + | WsAgentFinishedEvent + | WsSessionEndEvent + | WsUserTranscriptEvent + | WsAgentErrorEvent; /* Client → Server messages */ export interface WsStartSessionMessage { - type: "start-session"; - sessionType: "debate" | "conversation"; - config: any; + type: "start-session"; + sessionType: "debate" | "conversation"; + config: any; } export interface WsStopSessionMessage { - type: "stop-session"; + type: "stop-session"; } export interface WsStartConversationMessage { - type: "start-conversation"; - personaId: string; + type: "start-conversation"; + personaId: string; } export interface WsAudioChunkMessage { - type: "audio-chunk"; - chunk: string; + type: "audio-chunk"; + chunk: string; } export interface WsSpeechEndMessage { - type: "speech-end"; + type: "speech-end"; } export type WsClientMessage = - | WsStartSessionMessage - | WsStopSessionMessage - | WsStartConversationMessage - | WsAudioChunkMessage - | WsSpeechEndMessage; + | WsStartSessionMessage + | WsStopSessionMessage + | WsStartConversationMessage + | WsAudioChunkMessage + | WsSpeechEndMessage; /* Agent state tracked by the composable */ export interface AgentState { - speaking: boolean; - transcript: string; - audioTag: string; + speaking: boolean; + transcript: string; + audioTag: string; } diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index 4fbb06c..3ec970e 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -1,63 +1,64 @@ diff --git a/web-application/frontend/pages/session.vue b/web-application/frontend/pages/session.vue index 206e4a5..6af4981 100644 --- a/web-application/frontend/pages/session.vue +++ b/web-application/frontend/pages/session.vue @@ -1,131 +1,139 @@ diff --git a/web-application/frontend/stores/useSessionStore.ts b/web-application/frontend/stores/useSessionStore.ts index 48cfba8..96f1e48 100644 --- a/web-application/frontend/stores/useSessionStore.ts +++ b/web-application/frontend/stores/useSessionStore.ts @@ -2,173 +2,175 @@ import type { Agent, PersonaSummary, SessionStatus, SessionMode } from "~/models import type { SourceCard } from "~/models/source-card"; export const useSessionStore = defineStore("session", () => { - // ── State ────────────────────────────────────────────── - const topic = ref(""); - const mode = ref("conversation"); - const persona = ref(null); - const agents = ref([]); - const sources = ref([]); - const status = ref("idle"); - const currentSpeaker = ref(null); - const elapsedTime = ref(0); - const endReason = ref(""); - const userTranscript = ref(""); - const agentTranscript = ref(""); - - // ── Computed ─────────────────────────────────────────── - const sourcesReversed = computed(() => [...sources.value].reverse()); - const isActive = computed(() => status.value === "active"); - const agentById = computed(() => new Map(agents.value.map(a => [a.id, a]))); - const isConversation = computed(() => mode.value === "conversation"); - - // ── Timer (private) ─────────────────────────────────── - let timerInterval: ReturnType | null = null; - - function startTimer() { - stopTimer(); - timerInterval = setInterval(() => { - elapsedTime.value++; - }, 1000); - } - - function stopTimer() { - if (timerInterval) { - clearInterval(timerInterval); - timerInterval = null; + // ── State ────────────────────────────────────────────── + const topic = ref(""); + const mode = ref("conversation"); + const persona = ref(null); + const agents = ref([]); + const sources = ref([]); + const status = ref("idle"); + const currentSpeaker = ref(null); + const elapsedTime = ref(0); + const endReason = ref(""); + const userTranscript = ref(""); + const agentTranscript = ref(""); + + // ── Computed ─────────────────────────────────────────── + const sourcesReversed = computed(() => [...sources.value].reverse()); + const isActive = computed(() => status.value === "active"); + const agentById = computed(() => new Map(agents.value.map((a) => [a.id, a]))); + const isConversation = computed(() => mode.value === "conversation"); + + // ── Timer (private) ─────────────────────────────────── + let timerInterval: ReturnType | null = null; + + function startTimer() { + stopTimer(); + timerInterval = setInterval(() => { + elapsedTime.value++; + }, 1000); } - } - - // ── Actions ──────────────────────────────────────────── - - /** - * Starts a debate session (legacy mode). - */ - function startSession(newTopic: string) { - mode.value = "debate"; - topic.value = newTopic; - persona.value = null; - agents.value = []; - sources.value = []; - status.value = "connecting"; - currentSpeaker.value = null; - elapsedTime.value = 0; - endReason.value = ""; - userTranscript.value = ""; - agentTranscript.value = ""; - startTimer(); - } - - /** - * Starts a conversation session with a historical persona. - */ - function startConversation(selectedPersona: PersonaSummary) { - mode.value = "conversation"; - topic.value = selectedPersona.name; - persona.value = selectedPersona; - agents.value = [{ - id: selectedPersona.id, - name: selectedPersona.name, - voiceId: "", - avatar: selectedPersona.avatar, - stance: "", - era: selectedPersona.era, - profession: selectedPersona.profession, - }]; - sources.value = []; - status.value = "connecting"; - currentSpeaker.value = null; - elapsedTime.value = 0; - endReason.value = ""; - userTranscript.value = ""; - agentTranscript.value = ""; - startTimer(); - } - - function setActive() { - status.value = "active"; - } - - function registerAgent(agentId: string) { - if (agents.value.some(a => a.id === agentId)) return; - const index = agents.value.length; - agents.value.push({ - id: agentId, - name: index === 0 ? "For" : "Against", - voiceId: "", - avatar: "", - stance: index === 0 ? "for" : "against", - }); - } - - function setSpeaker(agentId: string | null) { - currentSpeaker.value = agentId; - if (agentId) registerAgent(agentId); - } - - function addSource(card: SourceCard) { - sources.value = [...sources.value, card]; - } - - function setUserTranscript(text: string) { - userTranscript.value = text; - } - - function setAgentTranscript(text: string) { - agentTranscript.value = text; - } - - function endSession(reason: string) { - status.value = "ended"; - currentSpeaker.value = null; - endReason.value = reason; - stopTimer(); - } - - function resetSession() { - topic.value = ""; - mode.value = "conversation"; - persona.value = null; - agents.value = []; - sources.value = []; - status.value = "idle"; - currentSpeaker.value = null; - elapsedTime.value = 0; - endReason.value = ""; - userTranscript.value = ""; - agentTranscript.value = ""; - stopTimer(); - } - - return { - // State - topic, - mode, - persona, - agents, - sources, - status, - currentSpeaker, - elapsedTime, - endReason, - userTranscript, - agentTranscript, - - // Computed - sourcesReversed, - isActive, - agentById, - isConversation, - - // Actions - startSession, - startConversation, - setActive, - registerAgent, - setSpeaker, - addSource, - setUserTranscript, - setAgentTranscript, - endSession, - resetSession, - }; + + function stopTimer() { + if (timerInterval) { + clearInterval(timerInterval); + timerInterval = null; + } + } + + // ── Actions ──────────────────────────────────────────── + + /** + * Starts a debate session (legacy mode). + */ + function startSession(newTopic: string) { + mode.value = "debate"; + topic.value = newTopic; + persona.value = null; + agents.value = []; + sources.value = []; + status.value = "connecting"; + currentSpeaker.value = null; + elapsedTime.value = 0; + endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; + startTimer(); + } + + /** + * Starts a conversation session with a historical persona. + */ + function startConversation(selectedPersona: PersonaSummary) { + mode.value = "conversation"; + topic.value = selectedPersona.name; + persona.value = selectedPersona; + agents.value = [ + { + id: selectedPersona.id, + name: selectedPersona.name, + voiceId: "", + avatar: selectedPersona.avatar, + stance: "", + era: selectedPersona.era, + profession: selectedPersona.profession, + }, + ]; + sources.value = []; + status.value = "connecting"; + currentSpeaker.value = null; + elapsedTime.value = 0; + endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; + startTimer(); + } + + function setActive() { + status.value = "active"; + } + + function registerAgent(agentId: string) { + if (agents.value.some((a) => a.id === agentId)) return; + const index = agents.value.length; + agents.value.push({ + id: agentId, + name: index === 0 ? "For" : "Against", + voiceId: "", + avatar: "", + stance: index === 0 ? "for" : "against", + }); + } + + function setSpeaker(agentId: string | null) { + currentSpeaker.value = agentId; + if (agentId) registerAgent(agentId); + } + + function addSource(card: SourceCard) { + sources.value = [...sources.value, card]; + } + + function setUserTranscript(text: string) { + userTranscript.value = text; + } + + function setAgentTranscript(text: string) { + agentTranscript.value = text; + } + + function endSession(reason: string) { + status.value = "ended"; + currentSpeaker.value = null; + endReason.value = reason; + stopTimer(); + } + + function resetSession() { + topic.value = ""; + mode.value = "conversation"; + persona.value = null; + agents.value = []; + sources.value = []; + status.value = "idle"; + currentSpeaker.value = null; + elapsedTime.value = 0; + endReason.value = ""; + userTranscript.value = ""; + agentTranscript.value = ""; + stopTimer(); + } + + return { + // State + topic, + mode, + persona, + agents, + sources, + status, + currentSpeaker, + elapsedTime, + endReason, + userTranscript, + agentTranscript, + + // Computed + sourcesReversed, + isActive, + agentById, + isConversation, + + // Actions + startSession, + startConversation, + setActive, + registerAgent, + setSpeaker, + addSource, + setUserTranscript, + setAgentTranscript, + endSession, + resetSession, + }; }); From 8437785550302300c2d1f28e14a5701c5d60df42 Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 11:52:36 +0100 Subject: [PATCH 06/18] fix(cre): remove bc template --- .claude/settings.local.json | 8 +- .cursor/rules/general.mdc | 6 - .github/copilot-instructions.md | 383 ++---------------- .openai/Agents.md | 252 ++---------- README.md | 29 +- mobile-application/src/control/app-events.ts | 14 - .../src/style/screens/account-settings.tsx | 14 +- web-application/backend/CONFIG.md | 42 -- .../backend/database/mongo.auto.json | 16 +- .../backend/database/mysql.auto.sql | 12 - .../backend/database/postgres.auto.sql | 12 - .../backend/documentation/services.md | 55 --- .../backend/documentation/tests.md | 19 +- .../mocha/function-tests/test-blockchain.ts | 119 ------ .../test/mocha/test-tools/server-setup.ts | 38 +- web-application/frontend/README.md | 2 +- .../forms/users/ProfileDataForm.vue | 2 +- .../modals/ChangePasswordModalDialog.vue | 2 +- .../frontend/components/tables/UsersTable.vue | 4 +- .../frontend/models/constants/form.ts | 3 +- 20 files changed, 98 insertions(+), 934 deletions(-) delete mode 100644 web-application/backend/test/mocha/function-tests/test-blockchain.ts diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 8d7bc29..57523db 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -5,11 +5,9 @@ "Bash(npm install:*)", "Bash(npm audit:*)", "Bash(npm view:*)", - "Bash(find C:UsersUsuario__CoworkElevenlabsdlt-project-template -name .env* -type f)", - "Bash(grep -r @asanrom/smart-contract-wrapper C:UsersUsuario__CoworkElevenlabsdlt-project-template --include=*.json --include=*.ts)", - "Bash(find C:UsersUsuario__CoworkElevenlabsdlt-project-template -name build* -type f)", - "Bash(grep -r \"BLOCKCHAIN\\\\|WALLET\\\\|ETHEREUM\\\\|RPC_URL\\\\|PRIVATE_KEY\" C:UsersUsuario__CoworkElevenlabsdlt-project-template --include=*.md --include=*.example)", - "Bash(find /c/Users/Usuario/__Cowork/Elevenlabs/dlt-project-template/web-application/backend/src -type f -name *.ts)", + "Bash(find C:UsersUsuario__CoworkElevenlabs01-firecrawlDeadTalk -name .env* -type f)", + "Bash(find C:UsersUsuario__CoworkElevenlabs01-firecrawlDeadTalk -name build* -type f)", + "Bash(find /c/Users/Usuario/__Cowork/Elevenlabs/01-firecrawl/DeadTalk/web-application/backend/src -type f -name *.ts)", "Bash(xargs grep:*)" ] } diff --git a/.cursor/rules/general.mdc b/.cursor/rules/general.mdc index ec78bf9..b67f462 100644 --- a/.cursor/rules/general.mdc +++ b/.cursor/rules/general.mdc @@ -34,15 +34,9 @@ This document provides the general development rules that must be followed throu - **Language:** TypeScript - **ORM:** tsbean-orm (multi-database) - **Cache/Messaging:** Redis -- **Blockchain Interaction:** Ethers.js and @asanrom/smart-contract-wrapper - **Authentication:** Database-backed session tokens - **Email:** Nodemailer -### Smart Contracts (`smart-contracts`) -- **Language:** Solidity -- **Framework:** Hardhat -- **Libraries:** Ethers.js, TypeChain, @asanrom/smart-contract-wrapper - ### Mobile Application (`mobile-application`) - **Framework:** React Native with Expo - **Language:** TypeScript/TSX diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3590d5a..d97675e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,367 +1,44 @@ # Project Analysis and Development Guide -This document provides a comprehensive overview of the repository's structure, conventions, and established patterns. It is intended to be used as a primary guide and context prompt for any developer or AI assistant working on this project. +This document provides baseline guidance for contributors and AI assistants. ---- +## General rules -## 1. General Development Instructions +- Keep code clean and consistent. +- Do not manually edit generated API bindings. +- Keep backend logging centralized (no ad-hoc `console.log` in server code). +- Use shared helpers for HTTP responses and error handling. +- Follow naming conventions and existing project patterns. -These are high-level rules that must be followed across the project to maintain consistency and quality. More specific rules for each section are detailed below. +## Tech stack -- **Code Cleanliness:** Do not include icons, emojis, or emoticons in code comments, test descriptions, commit messages, or log messages. -- **Centralized Logging (Backend):** Do not use `console.log` for logging in the backend. All server-side logs must be routed through the centralized `Monitor` class found at `web-application/backend/src/monitor.ts`. Use the Monitor exclusively for server-side logs. -- **API Documentation for Code Generation:** - - **Generator Script:** The entire API client layer—including TypeScript definitions and calling functions—is generated by the `web-application/backend/scripts/gen-api-bindings.js` script. **Do not modify generated files manually.** The script works by parsing JSDoc comments in the backend controller files. - - **Required Action:** To add or modify an API endpoint, you must document it with a specific JSDoc format above the controller function in its corresponding file within `web-application/backend/src/controllers/api/`. - - **How to add/modify an endpoint:** - 1. Add `@typedef` blocks for request/response/error models. - 2. Add an endpoint JSDoc block (summary + `Binding:` + `@route` + `@group` + `@param` + `@returns` + `@security`). - 3. Run the generator script: `npm run update-api-bindings` (which executes `scripts/gen-api-bindings.js`). - - **Structure of Comments:** - 1. **Model Definitions (`@typedef`):** First, define any request, response, or error models in separate JSDoc blocks using `@typedef`. Each property is defined with `@property {{type}} {name}.required - {description}`. - 2. **Endpoint Definition:** After the models, use a final JSDoc block to define the endpoint. This includes a summary, a custom `Binding:` tag (which sets the generated function's name), and JSDoc tags like `@route`, `@group`, `@param`, `@returns`, and `@security`. - - **Reusing Types:** Before defining a new `@typedef`, always check if a suitable model already exists in other controller files to avoid duplication. - - **Example of the Correct JSDoc Format:** - ```javascript - /** - * @typedef UsernameChangeRequest - * @property {string} username.required - New username - * @property {string} password.required - Account password - * @property {string} tfa_token - Two factor authentication code (required if 2FA enabled) - */ +- Frontend: Nuxt, Vue 3, TypeScript +- Backend: Express, TypeScript, tsbean-orm +- Mobile: React Native, Expo +- Infrastructure: Docker, CI/CD scripts - /** - * @typedef UsernameChangeBadRequest - * @property {string} code.required - Error code: - * - WRONG_PASSWORD - * - USERNAME_INVALID - * - USERNAME_IN_USE - * - INVALID_TFA_CODE - */ +## Repository structure - /** - * Changes username - * Binding: ChangeUsername - * @route POST /account/username - * @group account - * @param {UsernameChangeRequest.model} request.body - Request body - * @returns {UsernameChangeBadRequest.model} 400 - Bad request - * @returns {void} 200 - Success - * @security AuthToken - */ - ``` -- **Centralized Error Handling (Backend):** Use the utility functions `sendError` and `sendSuccess` from `web-application/backend/src/utils/http-utils.ts` to format all API responses. This ensures error payloads and success responses are consistent across the application. -- **Follow Naming Conventions:** Adhere strictly to the naming conventions for files, classes, functions, and variables as outlined in the "Code Conventions" section. +- `web-application/backend`: API, business logic, models, tests +- `web-application/frontend`: web UI and client logic +- `mobile-application`: mobile app source +- `deployment-scripts`: deployment and operations scripts ---- +## API and codegen -## 2. Tech Stack +- API definitions are generated from backend JSDoc. +- After endpoint changes, run `npm run update-api-bindings`. +- Keep endpoint docs synchronized with runtime behavior. -- **Frontend (`web-application/frontend`):** - - **Framework:** Vue.js 3 (using `vue` package). - - **Language:** TypeScript. - - **Router:** Vue Router (using `vue-router` package). - - **State Management/Logic:** Vue 3 Composition API patterns. No global state library like Vuex/Pinia is used. State logic is managed in `src/control/` files like `auth.ts`. - - **Internationalization (i18n):** `vue-i18n` package. - - **Build Tool:** Vite (configured in `vite.config.js`). - - **Styling:** Plain CSS with CSS Variables for theming. Global styles are in `web-application/frontend/src/style/`, with key files like `main.css`, `theme.css`, and `variables.css`. +## Testing -- **Backend (`web-application/backend`):** - - **Framework:** Express.js (using `express` package). - - **Language:** TypeScript. - - **ORM / Database:** The project uses `tsbean-orm`, a multi-database ORM. It supports MongoDB, MySQL, and PostgreSQL via its drivers (`tsbean-driver-mongo`, `tsbean-driver-mysql`, `tsbean-driver-postgres`). The active database is determined by environment configuration. - - **Cache/Messaging:** Redis (using `redis` package). - - **Blockchain Interaction:** Ethers.js and `@asanrom/smart-contract-wrapper`. The latter is used to interact with generated TypeScript contract wrappers located in `web-application/backend/src/contracts/`. - - **Authentication:** Database-backed session tokens. A session document (defined in `src/models/users/session.ts`) with a unique token is stored in the database for each user login. - - **Emailing:** Nodemailer (using `nodemailer` package), with email templates in `src/emails/`. +- Backend tests: Mocha + Chai +- Frontend tests: project test runner +- Keep tests deterministic and focused on behavior -- **Smart Contracts (`smart-contracts`):** - - **Language:** Solidity. - - **Development Framework:** Hardhat. - - **Libraries & Tooling:** Ethers.js, TypeChain, and `@asanrom/smart-contract-wrapper`. A custom script (`src/generate-contract-wrappers.ts`) uses this library to generate strongly-typed TypeScript classes from the contract ABIs, which are then used by the backend. - -- **Mobile Application (`mobile-application`):** - - **Framework:** React Native (using `react` and `react-native` packages). - - **Toolkit:** Expo (using the `expo` package). - - **Language:** TypeScript / TSX. - - **Navigation:** React Navigation (using `@react-navigation/native`). - -- **Infrastructure & Deployment:** - - **Containerization:** Docker, Docker Compose. - - **CI/CD:** GitHub Actions (see `deployment-scripts/GITHUB_ACTIONS_CI.md`) and Jenkins (see `Jenkinsfile`). - - **Orchestration/Automation:** Ansible (see `deployment-scripts/auto-deploy.yml`). - -## 3. Repository Structure - -Below is a simplified directory tree of the project. -```plaintext -├── README.md -├── besu-test-node -│ ├── README.md -│ ├── docker-compose.yml -│ └── networkFiles -│ └── config -│ └── genesis.json -├── deployment-scripts -│ ├── DEPLOYMENT_GUIDE.md -│ ├── GITHUB_ACTIONS_CI.md -│ ├── README.md -│ ├── auto-deploy.yml -│ └── monitoring-tools -│ ├── README.md -│ ├── development -│ │ ├── README.md -│ │ └── docker-compose.yml -│ └── production -│ ├── README.md -│ └── docker-compose.yml -├── jenkins-utils -│ ├── mongodb -│ │ └── docker-compose.yml -│ └── redis -│ └── docker-compose.yml -├── mobile-application -│ ├── App.tsx -│ ├── app.config.js -│ ├── package.json -│ └── src -│ ├── api -│ ├── components -│ ├── control -│ ├── hooks -│ ├── locales -│ ├── style -│ └── utils -├── smart-contracts -│ ├── README.md -│ ├── package.json -│ ├── src -│ │ ├── config -│ │ ├── contract-wrappers -│ │ ├── tests -│ │ └── utils -│ └── tsconfig.json -└── web-application -├── README.md -├── backend -│ ├── package.json -│ ├── src -│ │ ├── config -│ │ ├── controllers -│ │ ├── contracts -│ │ ├── database -│ │ ├── emails -│ │ ├── locales -│ │ ├── models -│ │ ├── services -│ │ └── utils -│ └── test -│ └── mocha -│ ├── api-tests -│ ├── function-tests -│ └── test-tools -└── frontend -├── index.html -├── package.json -├── src -│ ├── api -│ ├── assets -│ ├── components -│ ├── control -│ ├── locales -│ ├── style -│ └── utils -└── vite.config.js -``` -- **`besu-test-node/`**: Configuration for a Hyperledger Besu test node. - - **Extensions:** `.yml`, `.json` - - **Example:** `docker-compose.yml` to run the node. -- **`deployment-scripts/`**: Scripts and guides for deployment and monitoring. - - **Extensions:** `.yml`, `.md` - - **Example:** `auto-deploy.yml` (Ansible playbook). -- **`mobile-application/`**: Source code for the React Native mobile app. - - **Extensions:** `.tsx`, `.ts`, `.json` - - **Example:** `src/components/screens/HomeScreen.tsx`. -- **`smart-contracts/`**: Smart contracts and their deployment/testing scripts. - - **Extensions:** `.sol`, `.ts` - - **Example:** `src/tests/test-example-contract.ts`. -- **`web-application/`**: The main web application. - - **`backend/`**: REST API, business logic, and database connection. - - **Extensions:** `.ts`, `.json` - - **Example:** `src/controllers/api/api-auth.ts`. - - **`frontend/`**: User interface built with Vue.js. - - **Extensions:** `.vue`, `.ts`, `.css` - - **Example:** `src/components/routes/HomePage.vue`. - -## 4. Code Conventions - -- **File Naming:** - - `kebab-case` is used for most files. - - **Examples:** `api-group-account.ts`, `main-layout.css`, `users-service.ts`. - - Vue components use `PascalCase`. - - **Example:** `HomePage.vue`, `ModalDialogContainer.vue`. - -- **Class, Function, and Variable Naming:** - - **Classes:** `PascalCase`. Example: `class UsersService`, `class Monitor`. - - **Functions & Methods:** `camelCase`. Example: `async function getUserProfile()`, `sendError()`. - - **Constants:** `UPPER_SNAKE_CASE`. Example: `const MONGO_DB_URL`. - - **Interfaces (TypeScript):** `PascalCase` without prefixes like `I`. Example: `interface UserProfile`. - -- **Code Formatting:** - - **Imports:** Grouped by origin (external libraries, project aliases). - - **Spacing:** Consistent spacing is used (4 spaces for indentation in the backend, 2 spaces in the frontend, as enforced by the respective Prettier/ESLint configurations). - - **Line Length:** Generally kept under 120 characters. - - **Trailing Commas:** Used at the end of multiline object property lists and arrays. - - **Semicolons:** Semicolons are used at the end of each statement in TypeScript. - -## 5. Logging - -- **Centralization:** All **backend** logging logic must go through the `Monitor` class, located at `web-application/backend/src/monitor.ts`. This class centralizes writing logs to the console and, optionally, to files. It is typically accessed via `req.monitor` in controllers. -- **Log Levels:** - - `debug`: For detailed information useful during development. - - `info`: For important application flow events (e.g., "Server started"). - - `warn`: For unexpected but non-critical situations. - - `error`: For captured errors that prevent an operation from completing. -- **Standard Format:** `[YYYY-MM-DD HH:mm:ss.SSS] [LEVEL] [TAG] Message` - - **Example:** `[2023-10-27 10:30:00.123] [ERROR] [API-AUTH] Login failed for user test@example.com` - -## 6. Error Handling - -- **Error Capturing (Backend):** - - **Controllers:** `try/catch` blocks are used within controller functions to catch synchronous and asynchronous errors. - - **Global Middleware:** In `web-application/backend/src/app.ts`, a global error-handling middleware is configured as the last `app.use()` to catch any unhandled errors from the controllers. - - **Request Errors:** Validators are used to check the `body`, `query`, and `params` of requests before processing them. - -- **Canonical Error Responses:** - - Error responses must follow a consistent format, managed by the `sendError` and `sendErrorMessage` functions from `web-application/backend/src/utils/http-utils.ts`. - - **HTTP Code:** The semantically correct HTTP status code is used (e.g., `400` for Bad Request, `401` for Unauthorized, `404` for Not Found, `500` for Internal Server Error). - - **Error Payload:** The response body is a JSON object with an `error` key. - - **Example:** `sendError(res, 'USERNAME_IN_USE', 400)` which results in `res.status(400).json({ error: 'USERNAME_IN_USE' })` - -## 7. API & Models - -- **API (Backend):** - - **Routes:** Routes are defined in the controllers and follow the pattern `/api/v1/{group}/{action}`. - - **Example:** `POST /api/v1/auth/login` - - **Controllers:** Located in `web-application/backend/src/controllers/api/`. Each file groups related functionalities (e.g., `api-auth.ts`, `api-account.ts`). - - **DTOs (Data Transfer Objects):** The interfaces defining the shape of request and response data are defined via JSDoc comments in the controller files. These comments are the single source of truth and are used by the `gen-api-bindings.js` script to generate the `definitions.ts` files for the frontend and backend tests. - -- **Models (Backend):** - - **Declaration:** Models are defined using a two-class pattern from the `tsbean-orm` library. - 1. A plain data class that extends `DataModel` to define the shape of the data (e.g., `class User extends DataModel`). - 2. A manager class that extends `Model` (e.g., `class UserModel extends Model`). - - **Configuration:** All database mapping, including the collection/table name, fields, types, validation rules, and indexes, is configured programmatically within the constructor of the manager class, not through decorators. - - **Location:** Models are located in `web-application/backend/src/models/`. They are organized into subfolders if necessary (e.g., `models/users/`). - - **Example:** `web-application/backend/src/models/users/user.ts` defines the `User` data class and the `UserModel` manager class. The `UserModel` constructor configures the 'users' collection and its fields using methods like `this.addField()` and `this.addIndex()`. - - -## 8. Frontend - -- **Folder Structure (`web-application/frontend/src/`):** - - **`components/`**: Contains all `.vue` components. - - **`routes/`**: Components that represent full pages, linked to a route. - - **`modals/`**: Modal dialog components. - - **`layout/`**: Structural components like `TopBar.vue`, `MenuSideBar.vue`. - - **`utils/`**: Generic and reusable components (e.g., `PasswordInput.vue`). - - **`style/`**: Global and thematic CSS stylesheets. - - **`api/`**: Logic for making calls to the backend API (mostly auto-generated). - - **`control/`**: Business logic and local application state (e.g., `auth.ts`, `wallets.ts`). - - **`locales/`**: JSON translation files for i18n. - - **`composables/`**: Reusable composition functions (e.g., `useTheme.ts`). - -- **Naming Conventions:** - - **Components:** `PascalCase.vue` (e.g., `UserProfilePage.vue`). - - **Stylesheets:** `kebab-case.css` (e.g., `main-layout.css`). - -## 9. Testing - -- **Testing Framework (Backend):** **Mocha** is used as the execution framework, and **Chai** is used for assertions. -- **Folder Structure:** - - `web-application/backend/test/mocha/` - - **`api-tests/`**: Integration tests that make HTTP calls to the API. - - **`function-tests/`**: Unit tests for specific functions and utilities. - - **`test-tools/`**: Utilities to help with writing tests (e.g., `api-tester.ts`). -- **File Naming Pattern:** - - `tests-api-{feature}.ts` for API tests. (e.g., `tests-api-auth.ts`) - - `test-{utility}.ts` for unit tests. (e.g., `test-aes.ts`) - ---- - -## 10. Modular Architecture & API Integration - -- **Boundaries:** - - Frontend/Mobile ↔ Backend via REST only. No direct DB access from clients. - -- **Bindings:** - - After changing backend endpoints/JSDoc, regenerate bindings using `npm run update-api-bindings`. - - Clients must use generated request functions (do not hand-write fetch calls to our API). - -- **Shared Utilities:** - - Prefer existing helpers in each module's `utils/` rather than reinventing functionality. - ---- - -## 11. Git Workflow & Commit Policy - -- **Commit Messages (Conventional Commits, English):** - - **Format:** `(): ` - - **Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`, etc. - - **Examples:** - - `feat(auth): add username change endpoint` - - `fix(api): return USERNAME_IN_USE error code` - -- **Branching:** - - `main`: production-ready at all times. - - `develop`: integration branch for next release (auto-deployed to dev/test). - - `feature/` and `fix/` from `develop`. - - `hotfix/` from `main` (merge back to both `main` and `develop`). - -- **Pull Requests:** - - Open PRs into `develop` (or `main` for hotfix). - - At least one reviewer required. - - CI (build, lint, tests) must pass. - - Address review comments; keep PRs small and focused. - - Squash or merge per repo policy; ensure clean history. - ---- - -## 12. CI/CD, Tooling & Scripts - -- **CI:** - - GitHub Actions runs on PRs and on pushes to `develop`/`main`. - - Pipelines build all modules and run tests/linting. Do not bypass checks. - -- **Common Tasks:** - - **Install:** `npm ci` (per module) - - **Lint:** `npm run lint` - - **Test:** `npm test` - - **Generate API bindings (backend → clients):** `npm run update-api-bindings` (script executes `scripts/gen-api-bindings.js` and updates `web-application/frontend/src/api/` and test definitions as configured) - - **Generate contract wrappers (in smart-contracts):** run `generate-contract-wrappers.ts` (see package scripts) - ---- - -## 13. Configuration & Secrets - -- Use environment variables and checked-in `.env.example` files. -- Never commit secrets/keys/passwords. -- If a secret is required, fetch from env or secret manager. Reject any suggestion to hard-code credentials. - ---- - -## 14. Besu Test Node - -- `besu-test-node/` provides a local Hyperledger Besu network configuration. -- Start with `docker-compose.yml`; customize `networkFiles/config/genesis.json` as needed for development. - ---- - -## 15. Quality Checklist (pre-PR) - -- Lint passes locally. -- Unit/integration tests added/updated and passing. -- API JSDoc updated for any endpoint changes; bindings regenerated. -- No `console.log` in backend; logging via Monitor. -- Errors returned with proper codes and standardized payloads. -- Commit messages follow Conventional Commits. -- Documentation/comments updated where necessary. +## Quality checklist +- Lint and format pass +- Tests pass in touched modules +- Docs updated for contract or behavior changes +- No secrets committed diff --git a/.openai/Agents.md b/.openai/Agents.md index 2c794cd..c62a154 100644 --- a/.openai/Agents.md +++ b/.openai/Agents.md @@ -1,233 +1,55 @@ -# Agents.md — Repository Guide for OpenAI Codex / Agents (Revised) +# Agents.md - Repository Guide -> Single source of truth for how an AI coding agent should understand, navigate, and modify this repository. Keep it in sync with the codebase. +Single source of truth for AI agents working on this repository. ---- +## 1) Mission -## 1) Mission & Scope +Build and maintain high quality backend, frontend, and mobile features. -**Goal:** Enable the agent to implement features, fix bugs, and maintain quality across backend, frontend, and smart contracts following the project’s conventions. +## 2) Repository map -**Non‑Goals:** Experimental code without tests, changing public APIs without docs, bypassing CI, or committing secrets. - -**Authoritative sources (read first):** - -* `copilot-instructions.md` (global rules) -* `controllers.instructions.md` (REST + JSDoc/Swagger + bindings) -* `models.instructions.md` (tsbean‑orm patterns) -* `services.instructions.md` (service layer) -* `utils.instructions.md` (utilities) -* `config.instructions.md` (config singleton + env parsing) -* `CONFIG.md` (env reference, server/DB/Redis/Mail/Blockchain) -* `DEV_GUIDE.md` (index to deeper docs) -* Smart contracts: `CONTRACTS_DOC.md`, `smart-contracts/README.md` -* Web app: `web-application/README.md`, backend `web-application/backend/README.md`, frontend `web-application/frontend/README.md` - -**Style baseline:** TypeScript (backend/frontend); Solidity (contracts); Vue 3 + Vite (frontend); Express (backend). - ---- - -## 2) Repository Map (high level) - -``` +```text . -├─ web-application/ -│ ├─ backend/ # Express + TS (API + Swagger) -│ │ ├─ src/{controllers,models,services,utils,config,...} -│ │ └─ test/mocha # API + unit tests -│ └─ frontend/ # Vue 3 + Vite + TS -├─ smart-contracts/ # Solidity + build/deploy/tests + wrappers -└─ deployment-scripts/# CI/CD + ops (if present) +|- web-application/ +| |- backend/ +| `- frontend/ +|- mobile-application/ +`- deployment-scripts/ ``` -**Boundaries:** Clients talk to backend via REST only. No direct DB access from clients. Contract interaction is through generated wrappers. - ---- - -## 3) Canonical Commands - -### Backend (web-application/backend) - -* Install: `npm install` -* Build: `npm run build` -* Dev: `npm run dev` -* Start: `npm start` -* Test: `npm test` -* Update translations: `npm run update-translations` -* Swagger: served at `/api-docs` when enabled -* Generate API bindings (backend → clients/tests): `npm run update-api-bindings` (from project scripts) - -### Frontend (web-application/frontend) - -* Install: `npm install` -* Dev: `npm run serve` -* Lint/format: `npm run prettier` -* Test: `npm test` -* Build: `npm run build` - -**Frontend env:** Use `.env` with `VITE__*` variables (e.g., `VITE__API_SERVER_HOST`, `VITE__RECAPTCHA_SITE_KEY`). - -### Smart contracts (smart-contracts) - -* Install: `npm install` -* Build: `npm run build` -* Deploy: `npm run deploy` -* Upgrade: `npm run deploy:upgrade` -* Test: `npm test` - -**Wrappers:** Generated in `src/contract-wrappers/`; copy to backend when required. - -### Docker (web-application) - -* Build image: `docker build --tag web-application-name .` -* Compose up: `docker compose up --build` - ---- - -## 4) Editing Rules (hard constraints) - -### Logging & errors - -* **Backend:** No `console.*`. Use `Monitor` abstraction. Error responses through helpers with stable `code` values. - -### Controllers (REST) - -* Thin: routing + validation + call services. -* Mandatory JSDoc per handler: `Binding:`, `@route`, `@group`, `@param` (`.path|.query|.body|.formData|.header`), `@returns` referencing `@typedef ... .model` types. -* Never access DB directly; call services. -* After JSDoc changes run **`npm run update-api-bindings`**. - -### Services - -* Singleton `getInstance()`; no I/O at import time. -* Orchestrate domain logic; use models’ finders and instance methods only. -* Throw errors with stable `code`; log via `Monitor`. - -### Models (tsbean‑orm) - -* One `DataModel` per file; call `this.init()` in constructor. -* Assign fields with `enforceType(...)`; add indexes/annotations explicitly. -* Provide a static `finder` factory. - -### Utilities - -* Stateless/pure; named exports; no side effects on import. - -### Config - -* Singleton. Parse only from `process.env`, validate, freeze. No dynamic reads spread across code. - -### Generated/external code - -* Do **not** edit generated API bindings or contract wrappers. Update sources + regenerate. - -### Security & privacy - -* No secrets in code or logs; avoid PII. Prefer IDs/metadata. - ---- - -## 5) Configuration Surfaces (reference) - -See `CONFIG.md` for exhaustive list. Key groups: - -* **General/Server:** ports, HTTPS, proxy, upload size, ACME challenge. -* **Logs:** toggles for request/info/debug/trace; optional ElasticSearch sink. -* **DB:** `DB_TYPE` ∈ {Mongo, MySQL, Postgres} and per‑driver settings. -* **Users/Auth:** initial admin, optional Google login. -* **Mail:** SMTP settings; disabled → logs emails as debug. -* **Captcha:** optional reCAPTCHA v3. -* **Redis:** optional cache and RT messaging. -* **FFmpeg:** paths for `ffmpeg/ffprobe`. -* **Blockchain:** node RPC URL/protocol, admin PK, contract addresses, event sync settings. -* **Tests:** separate DB URL for API tests. - ---- - -## 6) Smart Contracts (build/deploy/use) - -* Contracts are in `smart-contracts/contracts` (Solidity). -* Build artifacts/wrappers generated by `npm run build`. -* Deployment flows support upgradeable proxies (ERC1967) and custom deploy/initialize/upgrade hooks (see `src/contracts.ts`). -* Use `CONTRACTS_DOC.md` for ABI‑level docs; prefer wrappers for app code. -* Required env for deployment: node RPC URLs/keys as per `smart-contracts/README.md`. - ---- - -## 7) Testing Strategy - -* **Backend:** Mocha + Chai under `test/mocha/` (API + unit). Name tests `tests-api-*.ts` / `test-*.ts` per conventions. -* **Frontend:** `npm test`; keep component tests focused and deterministic. -* **Contracts:** tests in `smart-contracts/tests/`; use provided helpers (`runTest`, `assertEvent`, …). Ensure local/test network is running. - ---- - -## 8) API Documentation Contract - -* `@typedef` blocks define request/response/error models. -* Endpoint block includes: summary, `Binding`, `@route`, `@group`, parameters, `@returns`, optional `@security`. -* Body parameters reference typedefs via `.model` (e.g., `{LoginRequest.model} request.body.required`). -* Error model exposes string `code` with enumerated values in description. - ---- - -## 9) CI/CD & Branching - -* `main`: production‑ready. -* `develop`: integration for next release. -* CI runs build + lint + tests on PRs and pushes. -* Keep PRs small and focused; do not bypass checks. - ---- - -## 10) Conventional Commits - -`(): ` where type ∈ {feat, fix, docs, style, refactor, test, chore}. - -Examples: - -* `feat(auth): add username change endpoint` -* `fix(api): return USERNAME_IN_USE error code` - ---- - -## 11) Safety Rails for the Agent - -* Work **inside** the repo; never touch OS‑global locations. -* No destructive actions (DB drops, data resets) unless explicitly required. -* Don’t modify licensing/legal docs. -* **Flag** changes that affect public API, data migrations, auth, or security posture. +## 3) Core rules ---- +- Keep changes small and focused. +- Do not edit generated files manually. +- Regenerate bindings when API docs change. +- Avoid hardcoded secrets and sensitive data. +- Add or update tests for behavior changes. -## 12) Minimal Change Workflow (how the agent should work) +## 4) Common commands -1. Read this file + `*.instructions.md` + relevant READMEs/CONFIG. -2. Identify entry point(s) (controller/service/model or frontend component/composable). -3. Propose a minimal plan: files to touch, API impact, tests. -4. Implement: +### Backend - * Controllers → typedefs first → handler JSDoc → route wiring → call service. - * Services → validate → use models→ DTOs → throw typed errors. - * Models → fields/indexes/`init()` → serialization helpers. - * Frontend → use generated API bindings + `request-browser` wrapper. - * Contracts → adjust `src/contracts.ts` entries → (re)generate wrappers. -5. Regenerate & build (`update-api-bindings`, wrappers, app build). -6. Run tests; self‑review with checklist; open PR. +- `npm install` +- `npm run build` +- `npm run dev` +- `npm test` ---- +### Frontend -## 13) Example Flow: Add an endpoint +- `npm install` +- `npm run dev` +- `npm run build` +- `npm test` -1. Add/extend `@typedef` models near the controller. -2. Implement handler with full JSDoc (`Binding`, `@route`, params, `@returns`). -3. Service logic, input validation, typed errors, DTO out. -4. Update tests; `npm run update-api-bindings`; lint/tests; open PR. +### Mobile ---- +- `npm install` +- `npm run start` +- `npm test` -## 14) Maintenance +## 5) Change workflow -* Keep this file synchronized with `*.instructions.md`, READMEs, and `CONFIG.md`. -* When architecture changes (new service, driver, CI step), update **Commands**, **Rules**, and **Checklist**. -* Treat this file as an interface contract for AI tools and new contributors. +1. Read relevant docs and target files. +2. Implement the minimum required change. +3. Run format/lint/tests for touched modules. +4. Update docs when behavior or contracts change. diff --git a/README.md b/README.md index d639606..236e4ee 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,26 @@ -# Project template +# DeadTalk -This is a project template meant to be used to accelerate the initial setup and development steps of a new project. - -In order to use this template, download it as zip and put it into a new repository, then start developing using the provided components or adding new ones. +Application repository with web, mobile, and deployment tooling. ## Components -This template has the basic components of a DLT project, with some example code and documentation in form of README files. - -### Hyperledger Besu test node - -You can find a dockerized test Ethereum (Hyperledger Besu) node in the [besu-test-node](./besu-test-node/) folder. - -This node is meant to be used for development, tests and basic demos. It is a pre-configured node for smart contracts to be deployed and tested. - -### Smart contracts - -Any smart contracts the project requires should be placed in the [smart-contracts](./smart-contracts/) folder. - -The smart contracts are compiled and deployed using the [Solidity Compiler](https://github.com/ethereum/solidity) and deployed / tested using the [Smart contract wrapper](https://github.com/AgustinSRG/smart-contract-wrapper) library. - ### Web application The web application project is located in the [web-application](./web-application/) folder. -The web application uses a [NodeJS](https://nodejs.org/en) backend and a [Vue.js](https://vuejs.org/) frontend. +It includes: + +- Backend API (Node.js + TypeScript) +- Frontend app (Nuxt + Vue) ### Mobile application The mobile application project is located in the [mobile-application](./mobile-application/) folder. -The mobile application uses [React Native](https://reactnative.dev/) and [Expo](https://expo.dev/) in order to use the same codebase for both Android and IOS. +It uses React Native with Expo. ### Deployment scripts You can find deployment scripts in the [deployment-scripts](./deployment-scripts/) folder. -This includes scripts to deploy the web application on a Linux server and CI/CD scripts. +This includes CI/CD and infrastructure-related utilities. diff --git a/mobile-application/src/control/app-events.ts b/mobile-application/src/control/app-events.ts index 8a61fb0..9ba53ec 100644 --- a/mobile-application/src/control/app-events.ts +++ b/mobile-application/src/control/app-events.ts @@ -40,20 +40,6 @@ export type AppEventsMap = { */ "loaded-locale": (locale: string) => void; - /** - * Event sent when the wallets list changes - * You can obtain it via WalletsController - */ - "wallet-list-changed": () => void; - - /** - * Event sent when the selected wallet changes - * @param id The wallet ID - * @param address The wallet address - * @param name The wallet name - */ - "current-wallet-changed": (id: string, address: string, name: string) => void; - /** * Event sent when the theme changes * @param theme The theme name diff --git a/mobile-application/src/style/screens/account-settings.tsx b/mobile-application/src/style/screens/account-settings.tsx index 83160e4..a3bc3d0 100644 --- a/mobile-application/src/style/screens/account-settings.tsx +++ b/mobile-application/src/style/screens/account-settings.tsx @@ -99,22 +99,22 @@ const AccountSettingsScreensStyles = (theme: ColorTheme) => { flexDirection: "row", }, - walletContainer: { + profileCardContainer: { paddingBottom: 12, }, - walletContainerInner: { + profileCardContainerInner: { borderRadius: 16, backgroundColor: theme.areaBackground, padding: 16, }, - walletHeader: { + profileCardHeader: { flexDirection: "row", alignItems: "center", }, - walletIconContainer: { + profileIconContainer: { width: 48, height: 48, borderRadius: 24, @@ -124,7 +124,7 @@ const AccountSettingsScreensStyles = (theme: ColorTheme) => { alignItems: "center", }, - walletName: { + profileName: { fontFamily: ThemeConstants.fonts.main, color: theme.text, fontSize: 18, @@ -134,14 +134,14 @@ const AccountSettingsScreensStyles = (theme: ColorTheme) => { marginLeft: 8, }, - walletAddress: { + profileSecondaryText: { paddingVertical: 8, opacity: 0.8, fontFamily: ThemeConstants.fonts.main, color: theme.text, }, - walletOptionsRow: { + profileOptionsRow: { paddingBottom: 10, flexDirection: "row", justifyContent: "space-between", diff --git a/web-application/backend/CONFIG.md b/web-application/backend/CONFIG.md index d019614..59268b8 100644 --- a/web-application/backend/CONFIG.md +++ b/web-application/backend/CONFIG.md @@ -211,48 +211,6 @@ FFmpeg is used to encode images and videos, if needed. | `FFPROBE_PATH` | Path to the `ffprobe` binary. | | `FFMPEG_PATH` | Path to the `ffmpeg` binary. | -## Blockchain configuration - -### Ethereum node - -| Variable | Description | -| -------------------------- | -------------------------------------------------------------- | -| `BLOCKCHAIN_NODE_PROTOCOL` | Protocol to use to connect to the node. Can be `http` or `ws`. | -| `BLOCKCHAIN_NODE_RPC_URL` | JSON-RPC connection URL. Example: `http://localhost:8545` | - -### Private keys - -| Variable | Description | -| --------------------- | --------------------------------------------------------------------------------------- | -| `BLOCKCHAIN_PK_ADMIN` | Administrator private key to use for the platform to interact with the smart contracts. | - -### Smart contracts - -Set the smart contract addresses after you deployed the smart contracts. - -| Variable | Description | -| ------------------------------- | --------------------------------------------- | -| `CONTRACT_ADDR_ExampleContract` | Smart contract address for: `ExampleContract` | - -### Event synchronization - -The event synchronization system is meant to scan the blockchain, find events and index them in the database. - -Make sure to enable it only for a single server, if you have multiple, to prevent collisions. - -| Variable | Description | -| ----------------------------- | ------------------------------------------------------------------------------------- | -| `BLOCKCHAIN_SYNC` | Can be `YES` or `NO`. Set it to `YES` to enable event synchronization. | -| `BLOCKCHAIN_SYNC_FIRST_BLOCK` | First block to synchronize. Normally set to 0. Use it to skip blocks for long chains. | -| `BLOCKCHAIN_SYNC_RANGE` | Max number of blocks to synchronize in a single step. | -| `BLOCKCHAIN_SYNC_TIME` | Expected block time in milliseconds in order to wait for more blocks. | - -If you want to reset the synchronized event data, run the following command: - -```sh -npm run reset-blockchain-sync -``` - ## Tests | Variable | Description | diff --git a/web-application/backend/database/mongo.auto.json b/web-application/backend/database/mongo.auto.json index b98d4bc..598f700 100644 --- a/web-application/backend/database/mongo.auto.json +++ b/web-application/backend/database/mongo.auto.json @@ -23,10 +23,6 @@ { "collection": "users", "field": "id" - }, - { - "collection": "wallets", - "field": "id" } ], "indexes": [ @@ -79,16 +75,6 @@ "dir": "" } ] - }, - { - "collection": "wallets", - "unique": false, - "fields": [ - { - "name": "uid", - "dir": "" - } - ] } ] -} \ No newline at end of file +} diff --git a/web-application/backend/database/mysql.auto.sql b/web-application/backend/database/mysql.auto.sql index e0e5652..8b44875 100644 --- a/web-application/backend/database/mysql.auto.sql +++ b/web-application/backend/database/mysql.auto.sql @@ -70,15 +70,3 @@ CREATE TABLE `users` ( CREATE UNIQUE INDEX `ix_users_s_1` ON `users`(`username_lower_case`); CREATE UNIQUE INDEX `ix_users_s_2` ON `users`(`email`); CREATE INDEX `ix_users_s_3` ON `users`(`role`); - -CREATE TABLE `wallets` ( - `id` VARCHAR(255) PRIMARY KEY, - `uid` VARCHAR(255), - `name` VARCHAR(255), - `address` VARCHAR(255), - `encrypted_key` TEXT, - `password_hash` VARCHAR(255), - `password_salt` VARCHAR(255) -); - -CREATE INDEX `ix_wallets_s_1` ON `wallets`(`uid`); diff --git a/web-application/backend/database/postgres.auto.sql b/web-application/backend/database/postgres.auto.sql index c085e77..23d2b30 100644 --- a/web-application/backend/database/postgres.auto.sql +++ b/web-application/backend/database/postgres.auto.sql @@ -70,15 +70,3 @@ CREATE TABLE "users" ( CREATE UNIQUE INDEX "ix_users_s_1" ON "users"("username_lower_case"); CREATE UNIQUE INDEX "ix_users_s_2" ON "users"("email"); CREATE INDEX "ix_users_s_3" ON "users"("role"); - -CREATE TABLE "wallets" ( - "id" VARCHAR(255) PRIMARY KEY, - "uid" VARCHAR(255), - "name" VARCHAR(255), - "address" VARCHAR(255), - "encrypted_key" TEXT, - "password_hash" VARCHAR(255), - "password_salt" VARCHAR(255) -); - -CREATE INDEX "ix_wallets_s_1" ON "wallets"("uid"); diff --git a/web-application/backend/documentation/services.md b/web-application/backend/documentation/services.md index 09424f6..8cf8afd 100644 --- a/web-application/backend/documentation/services.md +++ b/web-application/backend/documentation/services.md @@ -41,61 +41,6 @@ Note: When adding new services, remember to update this document. Here is a list of existing services with a brief explanation on their function and usage. -### Blockchain events scan service - -Location: [src/services/blockchain-events-scan.ts](../src/services/blockchain-events-scan.ts). - -The blockchain events scan service is used to scan the blockchain in order to find events, and index them in the database for easier access for the controllers. - -Inside the `scan` method, you must add asynchronous calls to methods that scan the events of all the smart contracts you need, here is an example: - -```ts -// ... -export class BlockchainEventsScanner { - // ... - private async scan() { - // ... - // Add scanning methods below this line - await this.scanEventsExampleContract(rangeStart, rangeEnd); - - this.currentBlock = rangeEnd; - // ... - } - - private async scanEventsExampleContract(fromBlock: bigint, toBlock: bigint) { - const events = await BlockchainConfig.getExampleSmartContract().findEvents(fromBlock, toBlock); - - this.logEvents(events.events); - - for (let i = 0; i < events.length(); i++) { - const eventType = events.getEventType(i); - switch (eventType) { - case "ExampleEvent": - { - const ev = events.getExampleEvent(i); - const timestamp = await this.getBlockTimestamp(ev.event.log.blockNumber); - // Handle event. Eg: Store in database - // ... - } - break; - } - } - } -} -``` - -### Blockchain service - -Location: [src/services/blockchain-service.ts](../src/services/blockchain-service.ts). - -The Blockchain service is used to encapsulate interaction logic with the blockchain, for example: - - - Handle wallets - - Make transaction options - - Obtains smart contract wrappers - -For smart contract wrappers, bytecodes or other smart contract tools, use the [src/contracts](../src/contracts/) folder. - ### Captcha service Location: [src/services/captcha-service.ts](../src/services/captcha-service.ts). diff --git a/web-application/backend/documentation/tests.md b/web-application/backend/documentation/tests.md index e54a30e..aecdafa 100644 --- a/web-application/backend/documentation/tests.md +++ b/web-application/backend/documentation/tests.md @@ -84,15 +84,13 @@ Example: "use strict"; import assert from "assert"; -import Crypto from "crypto"; import { APITester } from '../test-tools/api-tester'; import { PreparedUser, TestUsers } from '../test-tools/models/users'; -import { ApiWallet } from '../test-tools/api-bindings/api-group-wallet'; -import { privateKeyToAddress } from '@asanrom/smart-contract-wrapper'; +import { ApiAuth } from '../test-tools/api-bindings/api-group-auth'; // Test group -describe("API/Wallets", () => { +describe("API/Auth", () => { let testUser1: PreparedUser; before(async () => { @@ -105,15 +103,9 @@ describe("API/Wallets", () => { // Tests - const randomPassword = Crypto.randomBytes(8).toString("hex"); - let wallet1: string; - - it('Should be able to create a wallet', async () => { - const r = await APITester.Test(ApiWallet.CreateWallet({ name: "Wallet 1", password: randomPassword }), testUser1.auth); - - assert.equal(r.name, "Wallet 1"); - - wallet1 = r.id; + it('Should be able to get current user data', async () => { + const r = await APITester.Test(ApiAuth.Me(), testUser1.auth); + assert.equal(r.username, testUser1.username); }); // ... @@ -125,7 +117,6 @@ describe("API/Wallets", () => { In order to run the API tests, you need: - A test MongoDB database. You must specify its connection URL in the `TEST_DB_MONGO_URL` environment variable. - - A test Ethereum node. You must specify the connection details in the [blockchain configuration](../CONFIG.md#blockchain-configuration). ### Skipping code for tests diff --git a/web-application/backend/test/mocha/function-tests/test-blockchain.ts b/web-application/backend/test/mocha/function-tests/test-blockchain.ts deleted file mode 100644 index f924cca..0000000 --- a/web-application/backend/test/mocha/function-tests/test-blockchain.ts +++ /dev/null @@ -1,119 +0,0 @@ -// Function tests - -"use strict"; - -import Crypto from "crypto"; -import assert from 'assert'; -import { ZERO_ADDRESS, isZeroAddress, validateAddress, compareAddresses, compareBytes32, normalizeAddress, normalizeBytes32, validateBytes32, quantityToDecimalString, validateQuantityDecimalString, parseQuantityDecimalString } from "../../../src/utils/blockchain"; - -// Test group -describe("Blockchain utilities", () => { - it('isZeroAddress', async () => { - assert.equal(validateAddress(ZERO_ADDRESS), true); - assert.equal(isZeroAddress(ZERO_ADDRESS), true); - assert.equal(isZeroAddress("0xAb6355Ca5BC17C886b95f61EE601dD149F5C31B6"), false); - }); - - it('validateAddress', async () => { - assert.equal(validateAddress("0x"), false); - assert.equal(validateAddress("0x00"), false); - assert.equal(validateAddress("0000000000000000000000000000000000000000"), true); - assert.equal(validateAddress("0x0000000000000000000000000000000000000000"), true); - assert.equal(validateAddress("0xAb6355Ca5BC17C886b95f61EE601dD149F5C31B6"), true); - }); - - it('normalizeAddress', async () => { - const randomAddress = Crypto.randomBytes(20).toString("hex").toUpperCase(); - - let res = normalizeAddress(randomAddress); - - assert.match(res, /^0x[0-9a-z]{40}$/); - }); - - it('compareAddresses', async () => { - const randomAddress1 = Crypto.randomBytes(20).toString("hex").toUpperCase(); - const randomAddress2 = Crypto.randomBytes(20).toString("hex").toUpperCase(); - - assert(compareAddresses(randomAddress1, "0x" + randomAddress1)); - assert(compareAddresses(randomAddress1.toLowerCase(), randomAddress1)); - assert(compareAddresses(randomAddress1, randomAddress2) === (randomAddress1 === randomAddress2)); - }); - - it('normalizeBytes32', async () => { - const randomBytes = Crypto.randomBytes(32).toString("hex").toUpperCase(); - - let res = normalizeBytes32(randomBytes); - - assert.match(res, /^0x[0-9a-z]{64}$/); - }); - - it('compareBytes32', async () => { - const bytes1 = Crypto.randomBytes(32).toString("hex").toUpperCase(); - const bytes2 = Crypto.randomBytes(32).toString("hex").toUpperCase(); - - assert(compareBytes32(bytes1, "0x" + bytes1)); - assert(compareAddresses(bytes1.toLowerCase(), bytes1)); - assert(compareAddresses(bytes1, bytes2) === (bytes1 === bytes2)); - }); - - it('validateBytes32', async () => { - const randomBytes = Crypto.randomBytes(32).toString("hex").toUpperCase(); - - assert(!validateBytes32(randomBytes)); - assert(!validateBytes32(randomBytes.toLowerCase())); - assert(!validateBytes32(normalizeBytes32(randomBytes) + "a")); - - assert(validateBytes32("0x" + randomBytes)); - assert(validateBytes32(normalizeBytes32(randomBytes))); - }); - - it('quantityToDecimalString', async () => { - const tests: {q: bigint, decimals: number, str: string}[] = [ - {q: BigInt("12345"), decimals: 0, str: "12345"}, - {q: BigInt("12345"), decimals: 1, str: "1234.5"}, - {q: BigInt("12345"), decimals: 2, str: "123.45"}, - {q: BigInt("12345"), decimals: 3, str: "12.345"}, - {q: BigInt("12345"), decimals: 5, str: "0.12345"}, - {q: BigInt("12345"), decimals: 7, str: "0.0012345"}, - ]; - - for (let test of tests) { - const r = quantityToDecimalString(test.q, test.decimals); - assert.equal(r, test.str); - } - }); - - it('parseQuantityDecimalString', async () => { - const tests: {q: bigint, decimals: number, str: string}[] = [ - {q: BigInt("12345"), decimals: 0, str: "12345"}, - {q: BigInt("12345"), decimals: 1, str: "1234.5"}, - {q: BigInt("12345"), decimals: 2, str: "123.45"}, - {q: BigInt("12345"), decimals: 3, str: "12.345"}, - {q: BigInt("12345"), decimals: 5, str: "0.12345"}, - {q: BigInt("12345"), decimals: 7, str: "0.0012345"}, - ]; - - for (let test of tests) { - const r = parseQuantityDecimalString(test.str, test.decimals); - assert.equal(r, test.q); - } - }); - - it('validateQuantityDecimalString', async () => { - const tests: {valid: boolean, decimals: number, str: string}[] = [ - {valid: true, decimals: 0, str: "12345"}, - {valid: true, decimals: 1, str: "1234.5"}, - {valid: true, decimals: 2, str: "123.45"}, - {valid: true, decimals: 3, str: "12.345"}, - {valid: true, decimals: 5, str: "0.12345"}, - {valid: true, decimals: 7, str: "0.0012345"}, - {valid: false, decimals: 4, str: "0.0012345"}, - {valid: false, decimals: 4, str: "Invalid"}, - ]; - - for (let test of tests) { - const r = validateQuantityDecimalString(test.str, test.decimals); - assert.equal(r, test.valid); - } - }); -}); \ No newline at end of file diff --git a/web-application/backend/test/mocha/test-tools/server-setup.ts b/web-application/backend/test/mocha/test-tools/server-setup.ts index b026448..9a141aa 100644 --- a/web-application/backend/test/mocha/test-tools/server-setup.ts +++ b/web-application/backend/test/mocha/test-tools/server-setup.ts @@ -1,5 +1,5 @@ // Server setup -// Setups config, smart contracts and a test server +// Setups config and a test server "use strict"; @@ -9,12 +9,10 @@ import { AsyncSemaphore } from "@asanrom/async-tools"; import { TestLog } from "./log"; import { setupMongoDatabaseIndexes } from "../../../src/utils/mongo-db-indexes"; import { MainWebApplication } from "../../../src/app"; -import { BlockchainEventsScanner } from "../../../src/services/blockchain-events-scan"; import { Config } from "../../../src/config/config"; import { FileStorageConfig } from "../../../src/config/config-file-storage"; import { DatabaseConfig } from "../../../src/config/config-database"; import { TaskService } from "../../../src/services/task-service"; -import { Web3RPCClient } from "@asanrom/smart-contract-wrapper"; export interface TestServerState { port: number; @@ -64,12 +62,6 @@ export async function setupTestServer(): Promise { return SETUP_STATE.state; } -const TEST_BESU_NODE_RPC = "http://localhost:8545" -const TEST_PRIVATE_KEY = "3de106f01f3fa595f215f50a0daf2ddd1bd061663b69396783a70dcee9f1f755"; -const TEST_PRIVATE_KEY_BUFFER = Buffer.from(TEST_PRIVATE_KEY, "hex"); - -export const TEST_BINANCE_KEY = "jzfohdbb0tte42sgfzj9fkmpb5hmsswwnxbjy8waohjpytndpwck8lgpg5xxxitb"; - async function setupAll() { // Initial config @@ -101,21 +93,6 @@ async function setupAll() { process.env.LOG_ELASTIC_SEARCH_ENABLED = "NO"; process.env.FILE_STORAGE_PRIVATE_SECRET = "secret"; - // Set blockchain synced block to the current block - - const lastBlock = await Web3RPCClient.getInstance().getBlockByNumber("latest", { rpcURL: TEST_BESU_NODE_RPC }); - - if (lastBlock) { - process.env.BLOCKCHAIN_SYNC_FIRST_BLOCK = lastBlock.number + ""; - TestLog.info(`Set BLOCKCHAIN_SYNC_FIRST_BLOCK to ${lastBlock.number + ""}`); - } - - // Deploy smart contracts - - TestLog.info("Deploying test smart contracts..."); - - await deploySmartContracts(); - // Load config Config.getInstance(); @@ -139,19 +116,6 @@ async function setupAll() { // Start services TestLog.info("Starting rest of services..."); - // Run blockchain sync - await BlockchainEventsScanner.getInstance().start(); - // Start task service TaskService.getInstance().start(); } - - -async function deploySmartContracts() { - // Deploy smart contracts - // We assume they are already compiled - - - - // Change configuration (env) -} diff --git a/web-application/frontend/README.md b/web-application/frontend/README.md index ba9abc4..4f039f8 100644 --- a/web-application/frontend/README.md +++ b/web-application/frontend/README.md @@ -1097,7 +1097,7 @@ To create a custom modal, leverage the `ModalDialog` component. It is recommende @update:modelValue="updateModelValue" > - + diff --git a/web-application/frontend/components/modals/ChangePasswordModalDialog.vue b/web-application/frontend/components/modals/ChangePasswordModalDialog.vue index a02575a..38f7678 100644 --- a/web-application/frontend/components/modals/ChangePasswordModalDialog.vue +++ b/web-application/frontend/components/modals/ChangePasswordModalDialog.vue @@ -6,7 +6,7 @@ @update:modelValue="updateModelValue" > - + Date: Sat, 21 Mar 2026 12:02:25 +0100 Subject: [PATCH 07/18] fix(frontend): test --- .../frontend/components/tables/UsersTable.vue | 4 +--- .../frontend/composables/useAudioPlayback.ts | 3 +-- web-application/frontend/composables/useVoiceInput.ts | 2 +- web-application/frontend/control/app-preferences.ts | 2 +- web-application/frontend/eslint.config.mjs | 1 + web-application/frontend/models/session.ts | 10 ---------- web-application/frontend/pages/index.vue | 4 ++-- web-application/frontend/pages/session.vue | 4 ++-- web-application/frontend/stores/useSessionStore.ts | 3 ++- 9 files changed, 11 insertions(+), 22 deletions(-) diff --git a/web-application/frontend/components/tables/UsersTable.vue b/web-application/frontend/components/tables/UsersTable.vue index 3834dfb..ce57874 100644 --- a/web-application/frontend/components/tables/UsersTable.vue +++ b/web-application/frontend/components/tables/UsersTable.vue @@ -142,9 +142,7 @@ v-if="isEmpty || isFilteredEmpty" :hasContainer="true" :title="isFilteredEmpty ? $t('No results found') : $t('No users available')" - :description=" - isFilteredEmpty ? $t('Try adjusting your filtering settings') : $t('There are currently no users to display.') - " + :description="isFilteredEmpty ? $t('Try adjusting your filtering settings') : $t('There are currently no users to display.')" :icon="isFilteredEmpty ? 'mdi:magnify' : 'mdi:account-group-outline'" /> diff --git a/web-application/frontend/composables/useAudioPlayback.ts b/web-application/frontend/composables/useAudioPlayback.ts index 04e9b7b..ee37c25 100644 --- a/web-application/frontend/composables/useAudioPlayback.ts +++ b/web-application/frontend/composables/useAudioPlayback.ts @@ -99,7 +99,6 @@ export function useAudioPlayback() { processingQueue = false; if (playbackQueue.length > 0) { void processQueue(); - return; } if (!currentSource && playbackQueue.length === 0) { isPlaying.value = false; @@ -117,7 +116,7 @@ export function useAudioPlayback() { if (currentSource) { try { currentSource.stop(); - } catch (_e) { + } catch { // Ignore if already stopped } currentSource = null; diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts index 3fcd64d..95ccf41 100644 --- a/web-application/frontend/composables/useVoiceInput.ts +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -78,7 +78,7 @@ export function useVoiceInput(sendMessage: (msg: any) => void) { if (vadInstance) { try { vadInstance.destroy(); - } catch (_e) { + } catch { // Ignore cleanup errors } vadInstance = null; diff --git a/web-application/frontend/control/app-preferences.ts b/web-application/frontend/control/app-preferences.ts index 4a67168..ea0111c 100644 --- a/web-application/frontend/control/app-preferences.ts +++ b/web-application/frontend/control/app-preferences.ts @@ -1,6 +1,6 @@ // App preferences -import { fetchFromLocalStorage, fetchFromLocalStorageCache, saveIntoLocalStorage } from "@/utils/local-storage"; +import { fetchFromLocalStorageCache, saveIntoLocalStorage } from "@/utils/local-storage"; import { AppEvents } from "./app-events"; export type ColorThemeName = "light" | "dark"; diff --git a/web-application/frontend/eslint.config.mjs b/web-application/frontend/eslint.config.mjs index f0cd0ac..65234f2 100644 --- a/web-application/frontend/eslint.config.mjs +++ b/web-application/frontend/eslint.config.mjs @@ -5,6 +5,7 @@ import withNuxt from "./.nuxt/eslint.config.mjs"; export default withNuxt({ rules: { + "vue/html-self-closing": ["warn", { html: { void: "any" } }], "vue/attribute-hyphenation": "off", "vue/no-multiple-template-root": "off", "vue/require-default-prop": "off", diff --git a/web-application/frontend/models/session.ts b/web-application/frontend/models/session.ts index 18cf848..638b651 100644 --- a/web-application/frontend/models/session.ts +++ b/web-application/frontend/models/session.ts @@ -12,15 +12,5 @@ export interface Agent { profession?: string; } -export interface PersonaSummary { - id: string; - name: string; - era: string; - nationality: string; - profession: string; - avatar: string; - firstMessage: string; -} - export type SessionStatus = "idle" | "connecting" | "active" | "ended"; export type SessionMode = "debate" | "conversation"; diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index 3ec970e..48926f9 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -64,7 +64,7 @@ + + diff --git a/web-application/frontend/components/source-cards/SourceCardItem.vue b/web-application/frontend/components/source-cards/SourceCardItem.vue index 47580e1..3a36ae7 100644 --- a/web-application/frontend/components/source-cards/SourceCardItem.vue +++ b/web-application/frontend/components/source-cards/SourceCardItem.vue @@ -2,7 +2,7 @@ diff --git a/web-application/frontend/components/toggles/ThemeToggle.vue b/web-application/frontend/components/toggles/ThemeToggle.vue index b69efc9..df4809d 100644 --- a/web-application/frontend/components/toggles/ThemeToggle.vue +++ b/web-application/frontend/components/toggles/ThemeToggle.vue @@ -1,21 +1,7 @@ diff --git a/web-application/frontend/stores/useSessionStore.ts b/web-application/frontend/stores/useSessionStore.ts index cdb9ca7..d15a043 100644 --- a/web-application/frontend/stores/useSessionStore.ts +++ b/web-application/frontend/stores/useSessionStore.ts @@ -1,4 +1,4 @@ -import type { Agent, SessionStatus, SessionMode } from "~/models/session"; +import type { Agent, SessionStatus, SessionMode, MicMode, ConversationEntry } from "~/models/session"; import type { PersonaSummary } from "~/api/definitions"; import type { SourceCard } from "~/models/source-card"; @@ -15,6 +15,8 @@ export const useSessionStore = defineStore("session", () => { const endReason = ref(""); const userTranscript = ref(""); const agentTranscript = ref(""); + const micMode = ref("auto"); + const conversationHistory = ref([]); // ── Computed ─────────────────────────────────────────── const sourcesReversed = computed(() => [...sources.value].reverse()); @@ -56,6 +58,7 @@ export const useSessionStore = defineStore("session", () => { endReason.value = ""; userTranscript.value = ""; agentTranscript.value = ""; + conversationHistory.value = []; startTimer(); } @@ -84,6 +87,7 @@ export const useSessionStore = defineStore("session", () => { endReason.value = ""; userTranscript.value = ""; agentTranscript.value = ""; + conversationHistory.value = []; startTimer(); } @@ -120,6 +124,18 @@ export const useSessionStore = defineStore("session", () => { agentTranscript.value = text; } + function addConversationEntry(entry: ConversationEntry) { + conversationHistory.value = [...conversationHistory.value, entry]; + } + + function toggleMicMode() { + micMode.value = micMode.value === "auto" ? "manual" : "auto"; + } + + function setMicMode(newMode: MicMode) { + micMode.value = newMode; + } + function endSession(reason: string) { status.value = "ended"; currentSpeaker.value = null; @@ -139,6 +155,8 @@ export const useSessionStore = defineStore("session", () => { endReason.value = ""; userTranscript.value = ""; agentTranscript.value = ""; + micMode.value = "auto"; + conversationHistory.value = []; stopTimer(); } @@ -155,6 +173,8 @@ export const useSessionStore = defineStore("session", () => { endReason, userTranscript, agentTranscript, + micMode, + conversationHistory, // Computed sourcesReversed, @@ -173,5 +193,8 @@ export const useSessionStore = defineStore("session", () => { setAgentTranscript, endSession, resetSession, + addConversationEntry, + toggleMicMode, + setMicMode, }; }); From a59017c032b6e8e37676a1618c203875b9021a59 Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sat, 21 Mar 2026 19:19:45 +0100 Subject: [PATCH 09/18] fix(front):session --- web-application/frontend/pages/session.vue | 36 ++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/web-application/frontend/pages/session.vue b/web-application/frontend/pages/session.vue index 7b055cc..ba6bad0 100644 --- a/web-application/frontend/pages/session.vue +++ b/web-application/frontend/pages/session.vue @@ -199,6 +199,22 @@ class="w-5 h-5 text-[#d4a853]" /> + + + @@ -492,9 +508,13 @@ watch(isAgentPlaying, (playing) => { // Bridge: new sources → store watch(sourcesHistory, (history) => { - if (history.length > sources.value.length) { - const newSources = history.slice(sources.value.length); - newSources.forEach((s) => sessionStore.addSource(s)); + if (!history || history.length === 0) return; + const existingIds = new Set(sources.value.map((s) => s.id)); + for (const source of history) { + if (!existingIds.has(source.id)) { + sessionStore.addSource(source); + existingIds.add(source.id); + } } }); @@ -525,6 +545,16 @@ function onInterrupt() { sessionStore.setSpeaker(null); } +function onStop() { + stopListening(); + stopPlayback(); + wsSend({ type: "stop-session" }); +} + +function onBack() { + router.push(localePath("/")); +} + // Push-to-talk handlers function onPushStart() { startManualRecording(); From 2567cf9a545b975cce8071b4a74ef0e152ada9eb Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sun, 22 Mar 2026 11:29:35 +0100 Subject: [PATCH 10/18] feat: redesign and PTT --- mobile-application/src/api/definitions.ts | 30 ++ .../backend/src/config/personas.ts | 24 + .../src/controllers/api/api-personas.ts | 9 + .../src/controllers/api/api-session.ts | 107 ++++ .../src/controllers/websocket/websocket.ts | 8 + .../services/conversation-engine-service.ts | 26 +- .../test-tools/api-bindings/definitions.ts | 30 ++ web-application/frontend/api/definitions.ts | 30 ++ .../frontend/assets/css/seance-effects.css | 190 +++++++ .../components/seance/SeanceHeader.vue | 46 ++ .../components/session/SessionControls.vue | 247 +++++++++ .../session/SessionEndedOverlay.vue | 324 ++++++++++++ .../components/session/SessionPortrait.vue | 157 ++++++ .../session/SessionTranscriptOverlay.vue | 50 ++ .../composables/useOrchestratorSocket.ts | 5 +- .../frontend/composables/useVoiceInput.ts | 293 +++++------ web-application/frontend/i18n/locales/en.json | 105 ++-- web-application/frontend/i18n/locales/es.json | 105 ++-- web-application/frontend/layouts/seance.vue | 22 + web-application/frontend/pages/index.vue | 231 ++++++-- web-application/frontend/pages/session.vue | 497 +++++++++--------- 21 files changed, 1971 insertions(+), 565 deletions(-) create mode 100644 web-application/backend/src/controllers/api/api-session.ts create mode 100644 web-application/frontend/components/seance/SeanceHeader.vue create mode 100644 web-application/frontend/components/session/SessionControls.vue create mode 100644 web-application/frontend/components/session/SessionEndedOverlay.vue create mode 100644 web-application/frontend/components/session/SessionPortrait.vue create mode 100644 web-application/frontend/components/session/SessionTranscriptOverlay.vue create mode 100644 web-application/frontend/layouts/seance.vue diff --git a/mobile-application/src/api/definitions.ts b/mobile-application/src/api/definitions.ts index d625527..e1713eb 100644 --- a/mobile-application/src/api/definitions.ts +++ b/mobile-application/src/api/definitions.ts @@ -344,6 +344,21 @@ export interface PersonaSummary { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ @@ -398,6 +413,21 @@ export interface PersonaDetail { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ diff --git a/web-application/backend/src/config/personas.ts b/web-application/backend/src/config/personas.ts index 3a8d815..c4d450c 100644 --- a/web-application/backend/src/config/personas.ts +++ b/web-application/backend/src/config/personas.ts @@ -28,6 +28,9 @@ export interface Persona { firstMessagesEs: string[]; emotionalProfile: EmotionalTrigger[]; avatar: string; + image: string; + quote: string; + quoteEs: string; searchKeywords: string[]; } @@ -41,6 +44,9 @@ export interface PersonaSummary { nationality: string; profession: string; avatar: string; + image: string; + quote: string; + quoteEs: string; firstMessage: string; firstMessageEs: string; } @@ -160,6 +166,9 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "pause", trigger: "Wardenclyffe, unfulfilled dreams" }, ], avatar: "/images/personas/tesla.jpg", + image: "https://lh3.googleusercontent.com/aida-public/AB6AXuCfjbm1DU2EGnuA0SmkJrrBlgGcuTLGNepJliHb4i8T6zPjZT7xS7aUWI5gonnlXLhS8did-Wm6OkRgYlnwNjk_LdEQvbr9CYsxysx27a7OnFmm0UaHen00QLtf14Po9Xrh63ipHxx6nKyCrPNK25lKkB40EpknGi9YlDJA1e1HY4YwLSy0ltPTfq1fYJDV-A52qzp9fAHXCrK7T9i3f9x4Vgfp5br863N7G6LCZVtdtG1qPejUP9jS13a4iZZwCiZwqSd-9V-Pu8pG", + quote: "The present is theirs; the future, for which I really worked, is mine.", + quoteEs: "El presente es de ellos; el futuro, por el que realmente trabajé, es mío.", searchKeywords: ["electricity", "wireless", "edison", "alternating current", "inventor"], }, { @@ -255,6 +264,9 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "whispers", trigger: "Failed marriages, personal regrets, son Eduard" }, ], avatar: "/images/personas/einstein.jpg", + image: "https://lh3.googleusercontent.com/aida-public/AB6AXuD6Sg0caTpkA1Yz_yGd-bh2ri6EzC9_VMA4eA7qornPlfT1fcIngg96pVfc90Xxh4gNZNm3TFcwyueUvzCH5M1J_j7dxg80-mHR4gy0RRQtd1T6Sj6uzpR1GqHLlf639zJuXZNkgwNZAfZ5DCKwV_L4jpFkjQNXDTUnviaZcqTrdPWPHiXqQl23hGMxisj-_H1a9Pi42wTfI8w1AYH5O5YeSTFrflfTtMa-_W-bJEPmghjn7Fhq10VSdejjyxuqPkF3g9vJwSSt81_2", + quote: "Imagination is more important than knowledge. For knowledge is limited, whereas imagination embraces the entire world.", + quoteEs: "La imaginación es más importante que el conocimiento. Porque el conocimiento es limitado, mientras que la imaginación abarca el mundo entero.", searchKeywords: ["physics", "relativity", "quantum", "Nobel Prize", "pacifism"], }, { @@ -350,6 +362,9 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "pause", trigger: "Langevin scandal, Academy rejection" }, ], avatar: "/images/personas/curie.jpg", + image: "", + quote: "Nothing in life is to be feared, it is only to be understood.", + quoteEs: "Nada en la vida debe ser temido, solo debe ser comprendido.", searchKeywords: ["radioactivity", "radium", "polonium", "Nobel Prize", "women in science"], }, { @@ -446,6 +461,9 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "pause", trigger: "End of dynasty, fall of Egypt, children's fates" }, ], avatar: "/images/personas/cleopatra.jpg", + image: "https://lh3.googleusercontent.com/aida-public/AB6AXuBlKFbzesyN3MQW8yCX8b20xZZmWSlxVkyhxnARrmeFjFgmhqWUJfM_tKcaz5gDLCG96vNyhQ4n0jpN4Cd4VTEWtHXxMW7Y_Le-LtqdUcj232S5ISwZnTndgTYIxhfYw5HDMTXyuNtWuQ1njuhs1GYoNEzqFXUP4TeDxM0Hyvs2B__ORHl1AtS3LkAeBKa3xtlAUEcJeZZYfrlQOk5HZL6PwvjjN8NdarGMmIWedJ_UQBLJMZMh3H8ljGeqa_EP1ybHAB0I0Guf2-4r", + quote: "I will not be triumphed over.", + quoteEs: "No seré exhibida en triunfo.", searchKeywords: ["pharaoh", "Egypt", "Rome", "Antony", "Caesar", "Ptolemaic"], }, { @@ -543,6 +561,9 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "pause", trigger: "Legacy, returning to Apple, making a dent" }, ], avatar: "/images/personas/jobs.jpg", + image: "", + quote: "Stay hungry, stay foolish.", + quoteEs: "Sigue hambriento, sigue alocado.", searchKeywords: ["Apple", "iPhone", "design", "innovation", "Silicon Valley"], }, ]; @@ -594,6 +615,9 @@ export class PersonasConfig { nationality: p.nationality, profession: p.profession, avatar: p.avatar, + image: p.image, + quote: p.quote, + quoteEs: p.quoteEs, firstMessage: p.firstMessages.length > 0 ? p.firstMessages[Math.floor(Math.random() * p.firstMessages.length)] : p.firstMessage, diff --git a/web-application/backend/src/controllers/api/api-personas.ts b/web-application/backend/src/controllers/api/api-personas.ts index 7e447d6..82ea4cd 100644 --- a/web-application/backend/src/controllers/api/api-personas.ts +++ b/web-application/backend/src/controllers/api/api-personas.ts @@ -15,6 +15,9 @@ import { PersonasConfig } from "../../config/personas"; * @property {string} nationality - Nationality * @property {string} profession - Profession / title * @property {string} avatar - Avatar image path + * @property {string} image - Portrait image path + * @property {string} quote - Famous quote (English) + * @property {string} quoteEs - Famous quote (Spanish) * @property {string} firstMessage - Greeting message (English) * @property {string} firstMessageEs - Greeting message (Spanish) */ @@ -33,6 +36,9 @@ import { PersonasConfig } from "../../config/personas"; * @property {string} nationality - Nationality * @property {string} profession - Profession / title * @property {string} avatar - Avatar image path + * @property {string} image - Portrait image path + * @property {string} quote - Famous quote (English) + * @property {string} quoteEs - Famous quote (Spanish) * @property {string} firstMessage - Greeting message (English) * @property {string} firstMessageEs - Greeting message (Spanish) * @property {Array.} emotionalProfile - Emotional triggers @@ -93,6 +99,9 @@ export class PersonasController extends Controller { nationality: persona.nationality, profession: persona.profession, avatar: persona.avatar, + image: persona.image, + quote: persona.quote, + quoteEs: persona.quoteEs, firstMessage: persona.firstMessage, firstMessageEs: persona.firstMessageEs, emotionalProfile: persona.emotionalProfile, diff --git a/web-application/backend/src/controllers/api/api-session.ts b/web-application/backend/src/controllers/api/api-session.ts new file mode 100644 index 0000000..d065909 --- /dev/null +++ b/web-application/backend/src/controllers/api/api-session.ts @@ -0,0 +1,107 @@ +// Session API controller + +"use strict"; + +import Express from "express"; +import { noCache, sendApiError, sendApiResult } from "../../utils/http-utils"; +import { Controller } from "../controller"; +import { OpenAIService } from "../../services/openai-service"; +import { Monitor } from "../../monitor"; + +/** + * @typedef GenerateQuoteRequest + * @property {string} personaName.required - Name of the historical figure + * @property {Array.} transcript.required - Full conversation transcript + */ + +/** + * @typedef ConversationEntry + * @property {string} role.required - "user" or "agent" + * @property {string} text.required - Message text + */ + +/** + * @typedef GenerateQuoteResponse + * @property {string} quote - The generated memorable quote + */ + +/** + * @typedef GenerateQuoteError + * @property {string} code.required - Error code: + * - INVALID_INPUT: Missing or invalid input + * - GENERATION_FAILED: LLM failed to generate quote + */ + +/** + * Session API — endpoints for session-related operations. + * @group session - Session Operations + */ +export class SessionController extends Controller { + public registerAPI(prefix: string, application: Express.Express): any { + application.post(prefix + "/session/generate-quote", noCache(this.generateQuote.bind(this))); + } + + /** + * Generates a memorable quote from the conversation transcript using LLM. + * Takes the full conversation and distills it into a single evocative line + * that captures the essence of the exchange with the historical figure. + * Binding: GenerateQuoteRequest + * @route POST /session/generate-quote + * @group session + * @param {GenerateQuoteRequest.model} request.body.required - Transcript and persona info + * @returns {GenerateQuoteResponse.model} 200 - Generated quote + * @returns {GenerateQuoteError.model} 400 - Invalid input + */ + private async generateQuote(req: Express.Request, res: Express.Response) { + const { personaName, transcript } = req.body; + + if (!personaName || !transcript || !Array.isArray(transcript) || transcript.length === 0) { + return sendApiError(req, res, 400, "INVALID_INPUT"); + } + + try { + const openai = OpenAIService.getInstance(); + + // Build conversation text for context + const conversationText = transcript + .map((entry: { role: string; text: string }) => { + const speaker = entry.role === "agent" ? personaName : "User"; + // Clean audio tags + const cleanText = entry.text.replace(/\[[^\]]*\]/g, "").replace(/\s{2,}/g, " ").trim(); + return `${speaker}: ${cleanText}`; + }) + .join("\n"); + + const result = await openai.chat([ + { + role: "system", + content: `You are a literary curator distilling a séance conversation with ${personaName} into a single memorable quote. + +Rules: +- Extract or synthesize ONE evocative sentence (max 25 words) that captures the most profound, surprising, or emotionally resonant moment from the conversation. +- It should sound like something ${personaName} would actually say — match their historical voice, era, and intellectual style. +- Prefer poetic or philosophical phrasing over literal summaries. +- Do NOT use quotation marks in your response. +- Do NOT add attribution or explanation — just the quote itself. +- Write in the same language the conversation was held in.`, + }, + { + role: "user", + content: `Here is the full conversation transcript:\n\n${conversationText}\n\nDistill this into a single memorable quote from ${personaName}.`, + }, + ]); + + const quote = result.response?.trim() || ""; + + if (!quote) { + Monitor.warning("SessionController.generateQuote: empty response from LLM"); + return sendApiError(req, res, 500, "GENERATION_FAILED"); + } + + return sendApiResult(req, res, { quote }); + } catch (err) { + Monitor.exception(err, "SessionController.generateQuote failed"); + return sendApiError(req, res, 500, "GENERATION_FAILED"); + } + } +} diff --git a/web-application/backend/src/controllers/websocket/websocket.ts b/web-application/backend/src/controllers/websocket/websocket.ts index 539067a..0781880 100644 --- a/web-application/backend/src/controllers/websocket/websocket.ts +++ b/web-application/backend/src/controllers/websocket/websocket.ts @@ -72,8 +72,14 @@ export class WebsocketController { public message(msg: string | Buffer | ArrayBuffer) { if (!msg) { + Monitor.debug("WS message: empty message received"); return; } + const msgType = msg instanceof Buffer ? "Buffer" : msg instanceof ArrayBuffer ? "ArrayBuffer" : typeof msg; + const msgLen = typeof msg === "string" ? msg.length : (msg as any).byteLength || (msg as any).length || 0; + if (msgLen > 100) { + Monitor.debug("WS message: type=" + msgType + " len=" + msgLen); + } if (msg instanceof Buffer) { msg = msg.toString("utf8"); } @@ -169,6 +175,7 @@ export class WebsocketController { } break; case "audio-chunk": + Monitor.info("WS: audio-chunk received", { sessionId: this.sessionId, hasChunk: !!msg.chunk, chunkLength: msg.chunk?.length }); if (this.sessionId && msg.chunk) { ConversationEngineService.getInstance().handleAudioChunk( this.sessionId, @@ -177,6 +184,7 @@ export class WebsocketController { } break; case "speech-end": + Monitor.info("WS: speech-end received", { sessionId: this.sessionId }); if (this.sessionId) { ConversationEngineService.getInstance() .handleSpeechEnd(this.sessionId) diff --git a/web-application/backend/src/services/conversation-engine-service.ts b/web-application/backend/src/services/conversation-engine-service.ts index a244bc8..35ba152 100644 --- a/web-application/backend/src/services/conversation-engine-service.ts +++ b/web-application/backend/src/services/conversation-engine-service.ts @@ -158,12 +158,18 @@ export class ConversationEngineService { public handleAudioChunk(sessionId: string, chunk: string): void { const session = this.sessions.get(sessionId); if (!session) { + Monitor.warning("ConversationEngineService: handleAudioChunk - no session", { sessionId }); return; } try { const buffer = Buffer.from(chunk, "base64"); session.audioChunks.push(buffer); + Monitor.info("ConversationEngineService: audio chunk received", { + sessionId, + chunkBytes: buffer.length, + totalChunks: session.audioChunks.length, + }); } catch (err) { Monitor.warning("ConversationEngineService: invalid audio chunk", { sessionId, @@ -177,13 +183,21 @@ export class ConversationEngineService { * @param sessionId The session ID */ public async handleSpeechEnd(sessionId: string): Promise { + Monitor.info("ConversationEngineService: speech-end received", { + sessionId, + hasSession: this.sessions.has(sessionId), + isProcessing: this.sessions.get(sessionId)?.isProcessing, + audioChunks: this.sessions.get(sessionId)?.audioChunks?.length, + }); + const session = this.sessions.get(sessionId); if (!session) { + Monitor.warning("ConversationEngineService: handleSpeechEnd - no session", { sessionId }); return; } if (session.isProcessing) { - Monitor.debug( + Monitor.info( "ConversationEngineService: already processing, ignoring speech-end", { sessionId }, ); @@ -191,7 +205,7 @@ export class ConversationEngineService { } if (session.audioChunks.length === 0) { - Monitor.debug( + Monitor.info( "ConversationEngineService: no audio chunks, ignoring speech-end", { sessionId }, ); @@ -206,7 +220,7 @@ export class ConversationEngineService { try { // 1. Speech-to-Text (use session language for better recognition) - Monitor.debug("ConversationEngineService: STT", { + Monitor.info("ConversationEngineService: STT", { sessionId, audioBytes: audioBuffer.length, language: session.language, @@ -217,16 +231,16 @@ export class ConversationEngineService { ); if (!transcript || transcript.trim().length === 0) { - Monitor.debug("ConversationEngineService: empty transcript, skipping", { + Monitor.info("ConversationEngineService: empty transcript, skipping", { sessionId, }); session.isProcessing = false; return; } - Monitor.info("ConversationEngineService: user said", { + Monitor.info("ConversationEngineService: user transcript received", { sessionId, - transcript, + transcriptLength: transcript.length, }); // Emit user transcript to frontend diff --git a/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts b/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts index d625527..e1713eb 100644 --- a/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts +++ b/web-application/backend/test/mocha/test-tools/api-bindings/definitions.ts @@ -344,6 +344,21 @@ export interface PersonaSummary { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ @@ -398,6 +413,21 @@ export interface PersonaDetail { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ diff --git a/web-application/frontend/api/definitions.ts b/web-application/frontend/api/definitions.ts index d625527..e1713eb 100644 --- a/web-application/frontend/api/definitions.ts +++ b/web-application/frontend/api/definitions.ts @@ -344,6 +344,21 @@ export interface PersonaSummary { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ @@ -398,6 +413,21 @@ export interface PersonaDetail { */ avatar?: string; + /** + * Portrait image path + */ + image?: string; + + /** + * Famous quote (English) + */ + quote?: string; + + /** + * Famous quote (Spanish) + */ + quoteEs?: string; + /** * Greeting message (English) */ diff --git a/web-application/frontend/assets/css/seance-effects.css b/web-application/frontend/assets/css/seance-effects.css index 45113f7..b99649a 100644 --- a/web-application/frontend/assets/css/seance-effects.css +++ b/web-application/frontend/assets/css/seance-effects.css @@ -100,3 +100,193 @@ box-shadow: 0 0 0 8px rgba(212, 168, 83, 0); } } + +/* ── Atmospheric effects (session portal) ─────────────── */ + +/* Grain texture overlay — subtle noise for analog feel */ +.grain-overlay { + position: fixed; + inset: 0; + z-index: 50; + pointer-events: none; + opacity: 0.03; + background-image: url("data:image/svg+xml,%3Csvg viewBox='0 0 256 256' xmlns='http://www.w3.org/2000/svg'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.9' numOctaves='4' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)'/%3E%3C/svg%3E"); + background-repeat: repeat; + background-size: 256px 256px; +} + +/* Fog gradient — subtle gold radial glow centered */ +.fog-gradient { + position: fixed; + inset: 0; + z-index: 0; + pointer-events: none; + background: radial-gradient(circle at 50% 40%, rgba(242, 195, 107, 0.06) 0%, rgba(19, 19, 24, 0) 70%); +} + +/* Ambient orbs — large blurred decorative circles */ +.ambient-orb { + position: fixed; + border-radius: 9999px; + pointer-events: none; + z-index: 0; +} + +.ambient-orb--primary { + top: 25%; + right: -5rem; + width: 24rem; + height: 24rem; + background: rgba(242, 195, 107, 0.05); + filter: blur(120px); +} + +.ambient-orb--secondary { + bottom: 25%; + left: -5rem; + width: 16rem; + height: 16rem; + background: rgba(206, 193, 226, 0.05); + filter: blur(100px); +} + +/* Ectoplasm wave — blurred pulsing blobs behind portrait */ +.ectoplasm-wave { + filter: blur(40px); + mix-blend-mode: screen; + will-change: transform, opacity; +} + +/* Glassmorphism — frosted glass effect for floating panels */ +.glassmorphism { + background: rgba(42, 41, 47, 0.7); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 1px solid rgba(78, 70, 55, 0.15); +} + +/* ── Landing page effects ──────────────────────────────── */ + +/* Fog container — holds animated fog layers */ +.fog-container { + position: absolute; + inset: 0; + overflow: hidden; + pointer-events: none; + z-index: 1; +} + +.fog-layer { + position: absolute; + width: 200%; + height: 100%; + background: radial-gradient(circle at 50% 50%, rgba(242, 195, 107, 0.08) 0%, transparent 70%); + filter: blur(80px); + animation: fog-drift 30s linear infinite; +} + +@keyframes fog-drift { + from { + transform: translateX(-20%) translateY(-10%); + } + to { + transform: translateX(0%) translateY(0%); + } +} + +/* Hero radial glow */ +.hero-glow { + background: radial-gradient(circle at 50% 50%, rgba(242, 195, 107, 0.12) 0%, rgba(10, 10, 12, 0) 70%); +} + +/* Incantation text — glowing headline effect */ +.text-incantation { + text-shadow: 0 0 30px rgba(242, 195, 107, 0.25); + letter-spacing: -0.02em; +} + +/* Spirit card — glass panel for persona cards */ +.spirit-card { + background: linear-gradient(135deg, rgba(255, 255, 255, 0.03) 0%, rgba(255, 255, 255, 0.01) 100%); + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + border: 1px solid rgba(212, 168, 83, 0.12); +} + +/* Card aura — hover radial glow behind card */ +.card-aura { + background: radial-gradient(circle at center, rgba(242, 195, 107, 0.15) 0%, transparent 80%); + opacity: 0; + transition: opacity 0.7s ease; +} + +.spirit-card:hover .card-aura { + opacity: 1; +} + +/* Slow pulse animation */ +.animate-pulse-slow { + animation: pulse-slow 4s cubic-bezier(0.4, 0, 0.6, 1) infinite; +} + +@keyframes pulse-slow { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.5; + } +} + +/* Float animation */ +.animate-float { + animation: seance-float 6s ease-in-out infinite; +} + +@keyframes seance-float { + 0%, + 100% { + transform: translateY(0); + } + 50% { + transform: translateY(-10px); + } +} + +/* Ectoplasm pulse keyframes */ +@keyframes ectoplasm-breathe { + 0%, + 100% { + transform: scale(1); + opacity: 0.15; + } + 50% { + transform: scale(1.15); + opacity: 0.25; + } +} + +@keyframes ectoplasm-breathe-alt { + 0%, + 100% { + transform: scale(1.05); + opacity: 0.1; + } + 50% { + transform: scale(0.9); + opacity: 0.2; + } +} + +@keyframes ectoplasm-breathe-slow { + 0%, + 100% { + transform: scale(0.95); + opacity: 0.12; + } + 50% { + transform: scale(1.1); + opacity: 0.18; + } +} diff --git a/web-application/frontend/components/seance/SeanceHeader.vue b/web-application/frontend/components/seance/SeanceHeader.vue new file mode 100644 index 0000000..6fcd90d --- /dev/null +++ b/web-application/frontend/components/seance/SeanceHeader.vue @@ -0,0 +1,46 @@ + + + diff --git a/web-application/frontend/components/session/SessionControls.vue b/web-application/frontend/components/session/SessionControls.vue new file mode 100644 index 0000000..3ee9d93 --- /dev/null +++ b/web-application/frontend/components/session/SessionControls.vue @@ -0,0 +1,247 @@ + + + diff --git a/web-application/frontend/components/session/SessionEndedOverlay.vue b/web-application/frontend/components/session/SessionEndedOverlay.vue new file mode 100644 index 0000000..27c7cbc --- /dev/null +++ b/web-application/frontend/components/session/SessionEndedOverlay.vue @@ -0,0 +1,324 @@ + + + + + diff --git a/web-application/frontend/components/session/SessionPortrait.vue b/web-application/frontend/components/session/SessionPortrait.vue new file mode 100644 index 0000000..10beabd --- /dev/null +++ b/web-application/frontend/components/session/SessionPortrait.vue @@ -0,0 +1,157 @@ + + + diff --git a/web-application/frontend/components/session/SessionTranscriptOverlay.vue b/web-application/frontend/components/session/SessionTranscriptOverlay.vue new file mode 100644 index 0000000..614d318 --- /dev/null +++ b/web-application/frontend/components/session/SessionTranscriptOverlay.vue @@ -0,0 +1,50 @@ + + + diff --git a/web-application/frontend/composables/useOrchestratorSocket.ts b/web-application/frontend/composables/useOrchestratorSocket.ts index a86a054..f96218d 100644 --- a/web-application/frontend/composables/useOrchestratorSocket.ts +++ b/web-application/frontend/composables/useOrchestratorSocket.ts @@ -67,6 +67,7 @@ export function useOrchestratorSocket() { } catch { return; } + console.log("[WS] received:", msg.event); switch (msg.event) { case "hello": @@ -163,7 +164,9 @@ export function useOrchestratorSocket() { * @param message The message object to send */ function send(message: any) { - wsSend(JSON.stringify(message)); + const json = JSON.stringify(message); + console.log("[WS] send:", message.type, "payload size:", json.length, "ws status:", status.value); + wsSend(json); } return { diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts index 2195d23..3be7989 100644 --- a/web-application/frontend/composables/useVoiceInput.ts +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -2,7 +2,9 @@ * Composable for real-time voice input with Voice Activity Detection (VAD). * Supports two modes: * - "auto" (VAD): Automatically detects when the user speaks and stops speaking. - * - "manual" (push-to-talk): Records while the user holds the button, sends on release. + * - "manual" (push-to-talk): VAD always runs; PTT controls when audio is sent. + * On push-start: mark active + accumulate VAD segments. + * On push-end: pause VAD (forces onSpeechEnd flush) then send accumulated audio. * * Also supports echo suppression by pausing VAD while the agent is playing audio. */ @@ -21,14 +23,12 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: let mediaStream: MediaStream | null = null; let vadInstance: any = null; - // Manual recording state - let manualRecorder: ScriptProcessorNode | null = null; - let manualAudioContext: AudioContext | null = null; - let manualSource: MediaStreamAudioSourceNode | null = null; - let manualSamples: Float32Array[] = []; - let manualStartTime = 0; - let manualSampleRate = 48000; - let isManualRecording = false; + // PTT state — accumulates VAD-captured audio segments while button is held + let pttActive = false; + let pttStartTime = 0; + let pttAudioSegments: Float32Array[] = []; + let pttFlushResolve: (() => void) | null = null; + let pttWaitingForSpeechEnd = false; /** * Starts listening for voice input. @@ -39,7 +39,6 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: error.value = null; try { - // Request microphone access mediaStream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, @@ -49,7 +48,6 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: }, }); - // Dynamically import VAD to avoid SSR issues const { MicVAD } = await import("@ricky0123/vad-web"); const vadAssetBase = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/"; @@ -60,31 +58,41 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: onnxWASMBasePath: onnxWasmBase, getStream: async () => mediaStream!, onSpeechStart: () => { - isSpeaking.value = true; + const currentMode = micMode?.value || "auto"; + if (currentMode === "auto") { + isSpeaking.value = true; + } + // In manual mode: VAD speech detection still runs silently }, onSpeechEnd: (audio: Float32Array) => { - isSpeaking.value = false; - - // Convert Float32 PCM to 16-bit PCM WAV and send - const wavBuffer = float32ToWav(audio, 16000); - const base64 = arrayBufferToBase64(wavBuffer); - - // Send as a single chunk + speech-end - sendMessage({ type: "audio-chunk", chunk: base64 }); - sendMessage({ type: "speech-end" }); + const currentMode = micMode?.value || "auto"; + + if (currentMode === "auto") { + isSpeaking.value = false; + // Auto mode: send immediately + const wavBuffer = float32ToWav(audio, 16000); + const base64 = arrayBufferToBase64(wavBuffer); + sendMessage({ type: "audio-chunk", chunk: base64 }); + sendMessage({ type: "speech-end" }); + } else if (pttActive || pttWaitingForSpeechEnd) { + // Manual mode: accumulate segment + pttAudioSegments.push(new Float32Array(audio)); + // If waiting for flush (button released), resolve now + if (pttFlushResolve) { + pttFlushResolve(); + pttFlushResolve = null; + } + } }, - positiveSpeechThreshold: 0.85, - negativeSpeechThreshold: 0.4, - redemptionMs: 700, - preSpeechPadMs: 100, - minSpeechMs: 300, + positiveSpeechThreshold: 0.8, + negativeSpeechThreshold: 0.35, + redemptionMs: 150, + preSpeechPadMs: 200, + minSpeechMs: 150, }); - // Start VAD only if in auto mode (or no mode specified) - const currentMode = micMode?.value || "auto"; - if (currentMode === "auto") { - vadInstance.start(); - } + // VAD always runs — it captures audio for both modes + vadInstance.start(); isListening.value = true; } catch (err: any) { @@ -97,13 +105,17 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: * Stops listening and releases resources. */ function stopListening() { - cleanupManualRecording(false); + pttActive = false; + pttAudioSegments = []; + pttStartTime = 0; + pttFlushResolve = null; + pttWaitingForSpeechEnd = false; if (vadInstance) { try { vadInstance.destroy(); } catch { - // Ignore cleanup errors + /* ignore */ } vadInstance = null; } @@ -121,14 +133,11 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: if (isAgentPlaying) { watch(isAgentPlaying, (playing) => { if (!vadInstance || !isListening.value) return; - const currentMode = micMode?.value || "auto"; - if (currentMode !== "auto") return; if (playing) { try { vadInstance.pause(); } catch { - // Fallback: disable audio track if (mediaStream) { const track = mediaStream.getAudioTracks()[0]; if (track) track.enabled = false; @@ -138,7 +147,6 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: try { vadInstance.start(); } catch { - // Fallback: re-enable audio track if (mediaStream) { const track = mediaStream.getAudioTracks()[0]; if (track) track.enabled = true; @@ -148,34 +156,13 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: }); } - // ── Mode switching: pause/resume VAD when toggling auto/manual ── + // ── Mode switching ── if (micMode) { watch(micMode, (newMode, oldMode) => { - // If switching away from manual while recording, clean up gracefully - if (oldMode === "manual" && isManualRecording) { - cleanupManualRecording(true); - } - - if (!vadInstance || !isListening.value) return; - - if (newMode === "auto") { - // Resume VAD (unless agent is currently playing) - const agentPlaying = isAgentPlaying?.value || false; - if (!agentPlaying) { - try { - vadInstance.start(); - } catch { - // Already running - } - } - } else { - // Pause VAD for manual mode - try { - vadInstance.pause(); - } catch { - // Not running - } + if (oldMode === "manual" && pttActive) { + stopManualRecording(); } + // VAD always runs — no pause/resume needed on mode switch }); } @@ -183,113 +170,111 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: /** * Starts manual recording (push-to-talk). - * Captures raw PCM samples until stopManualRecording() is called. + * VAD keeps running and accumulating speech segments. */ function startManualRecording() { - if (!mediaStream || !isListening.value) return; - if (isManualRecording) return; // Already recording + if (!isListening.value) return; + if (pttActive) return; - manualSamples = []; - manualStartTime = Date.now(); - isManualRecording = true; + pttAudioSegments = []; + pttStartTime = Date.now(); + pttActive = true; + pttWaitingForSpeechEnd = false; isSpeaking.value = true; - try { - // Use default sample rate (browser-native) to avoid compatibility issues - manualAudioContext = new AudioContext(); - manualSampleRate = manualAudioContext.sampleRate; - manualSource = manualAudioContext.createMediaStreamSource(mediaStream); - - // ScriptProcessorNode to capture raw PCM (4096 buffer, mono) - manualRecorder = manualAudioContext.createScriptProcessor(4096, 1, 1); - manualRecorder.onaudioprocess = (event: AudioProcessingEvent) => { - if (!isManualRecording) return; - const inputData = event.inputBuffer.getChannelData(0); - manualSamples.push(new Float32Array(inputData)); - }; - - manualSource.connect(manualRecorder); - manualRecorder.connect(manualAudioContext.destination); - } catch (err: any) { - isManualRecording = false; - isSpeaking.value = false; - error.value = err.message || "Failed to start manual recording"; + // Ensure VAD is running + if (vadInstance) { + try { + vadInstance.start(); + } catch { + /* already running */ + } } } /** - * Stops manual recording and sends the captured audio. + * Stops manual recording. + * Does NOT pause VAD — lets it finish detecting the current speech naturally. + * Waits for onSpeechEnd to fire (user stopped talking → silence detected), + * then sends accumulated audio. */ function stopManualRecording() { - cleanupManualRecording(true); - } + if (!pttActive) return; - /** - * Cleans up manual recording resources. - * @param sendAudio If true, sends captured audio to the backend - */ - function cleanupManualRecording(sendAudio: boolean) { - if (!isManualRecording && manualSamples.length === 0) return; + const duration = Date.now() - pttStartTime; - const duration = Date.now() - manualStartTime; - const samples = [...manualSamples]; + if (duration < MIN_MANUAL_DURATION_MS) { + pttActive = false; + isSpeaking.value = false; + pttAudioSegments = []; + pttStartTime = 0; + return; + } - // Reset state immediately - isManualRecording = false; + // Keep accumulation window open until VAD emits onSpeechEnd + pttWaitingForSpeechEnd = true; isSpeaking.value = false; - manualSamples = []; - manualStartTime = 0; - // Disconnect and clean up ScriptProcessorNode - if (manualRecorder) { - try { - manualRecorder.disconnect(); - } catch { - // Ignore - } - manualRecorder = null; - } - if (manualSource) { - try { - manualSource.disconnect(); - } catch { - // Ignore - } - manualSource = null; - } - if (manualAudioContext) { + // Wait for VAD to fire onSpeechEnd (user stopped talking → silence) + const flushPromise = new Promise((resolve) => { + pttFlushResolve = resolve; + }); + + // Fallback: if VAD doesn't fire, force a VAD flush before finalizing. + const timeoutId = setTimeout(() => { + if (!pttFlushResolve) return; try { - manualAudioContext.close(); + vadInstance?.pause(); } catch { - // Ignore + /* ignore */ } - manualAudioContext = null; - } - - if (!sendAudio) return; + setTimeout(() => { + if (pttFlushResolve) { + pttFlushResolve(); + pttFlushResolve = null; + } + try { + if (!isAgentPlaying?.value) { + vadInstance?.start(); + } + } catch { + /* ignore */ + } + }, 250); + }, 3000); + + flushPromise.then(() => { + clearTimeout(timeoutId); + pttActive = false; + pttWaitingForSpeechEnd = false; + pttFlushResolve = null; + sendAccumulatedAudio(); + }); + } - // Check minimum duration - if (duration < MIN_MANUAL_DURATION_MS) return; + /** + * Sends all accumulated PTT audio segments as a single WAV. + */ + function sendAccumulatedAudio() { + const segments = [...pttAudioSegments]; + pttAudioSegments = []; + pttStartTime = 0; - // Merge all captured samples into a single Float32Array - if (samples.length === 0) return; + if (segments.length === 0) return; - const totalLength = samples.reduce((sum, arr) => sum + arr.length, 0); + // Merge all VAD-captured segments into a single Float32Array + const totalLength = segments.reduce((sum, arr) => sum + arr.length, 0); if (totalLength === 0) return; const merged = new Float32Array(totalLength); let offset = 0; - for (const chunk of samples) { - merged.set(chunk, offset); - offset += chunk.length; + for (const segment of segments) { + merged.set(segment, offset); + offset += segment.length; } - // Resample to 16kHz if needed (browser may give 44.1kHz or 48kHz) - const targetRate = 16000; - const resampled = manualSampleRate !== targetRate ? resampleLinear(merged, manualSampleRate, targetRate) : merged; - - // Convert to WAV and send - const wavBuffer = float32ToWav(resampled, targetRate); + // VAD produces 16kHz audio — encode as WAV and send + const wavBuffer = float32ToWav(merged, 16000); const base64 = arrayBufferToBase64(wavBuffer); sendMessage({ type: "audio-chunk", chunk: base64 }); @@ -312,24 +297,6 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: }; } -/** - * Simple linear resampling from one sample rate to another. - */ -function resampleLinear(samples: Float32Array, fromRate: number, toRate: number): Float32Array { - if (fromRate === toRate) return samples; - const ratio = fromRate / toRate; - const newLength = Math.round(samples.length / ratio); - const result = new Float32Array(newLength); - for (let i = 0; i < newLength; i++) { - const srcIndex = i * ratio; - const srcFloor = Math.floor(srcIndex); - const srcCeil = Math.min(srcFloor + 1, samples.length - 1); - const frac = srcIndex - srcFloor; - result[i] = samples[srcFloor] * (1 - frac) + samples[srcCeil] * frac; - } - return result; -} - /** * Converts a Float32Array of PCM samples to a WAV file ArrayBuffer. */ @@ -343,26 +310,22 @@ function float32ToWav(samples: Float32Array, sampleRate: number): ArrayBuffer { const buffer = new ArrayBuffer(headerSize + dataSize); const view = new DataView(buffer); - // RIFF header writeString(view, 0, "RIFF"); view.setUint32(4, 36 + dataSize, true); writeString(view, 8, "WAVE"); - // fmt sub-chunk writeString(view, 12, "fmt "); - view.setUint32(16, 16, true); // Sub-chunk size - view.setUint16(20, 1, true); // PCM format + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); view.setUint16(22, numChannels, true); view.setUint32(24, sampleRate, true); view.setUint32(28, byteRate, true); view.setUint16(32, blockAlign, true); view.setUint16(34, bitsPerSample, true); - // data sub-chunk writeString(view, 36, "data"); view.setUint32(40, dataSize, true); - // Write samples as 16-bit PCM let offset = 44; for (let i = 0; i < samples.length; i++) { const s = Math.max(-1, Math.min(1, samples[i])); diff --git a/web-application/frontend/i18n/locales/en.json b/web-application/frontend/i18n/locales/en.json index c50533f..209d605 100644 --- a/web-application/frontend/i18n/locales/en.json +++ b/web-application/frontend/i18n/locales/en.json @@ -7,12 +7,12 @@ "Access denied. You lack the required permission to visit this page.": "", "Account": "", "Account created": "", + "Account info": "", "Account settings": "", "Action denied. You lack the required permission to perform this action.": "", "Active sessions": "", "Admin": "", "Admin panel": "", - "Against": "", "All banned status": "", "All dates": "", "All roles": "", @@ -20,6 +20,8 @@ "An unknown error occurred": "", "Application": "", "April": "", + "Archival References": "", + "Archive Verified": "", "Are you sure you want to close all sessions?": "", "Are you sure you want to close this session?": "", "Are you sure you want to delete this role?": "", @@ -31,19 +33,20 @@ "August": "", "Avatar": "", "Avatar removed successfully": "", - "Back to home": "", "Back to home page": "", "Back to homepage": "", "Ban users": "", "Banned": "", + "Begin the Séance": "", "Both passwords must match.": "", "Browser": "", + "Built with ElevenLabs + Firecrawl for #ElevenHacks": "", "Cancel": "", + "Change account password": "", "Change password": "", "Changes saved": "", - "Check secret key": "", "Check your inbox for an email we have sent to your new email in order to verify it and complete the change.": "", - "Checking": "", + "Choose a consciousness to tether": "", "Close": "", "Close all sesions": "", "Close all session": "", @@ -52,7 +55,9 @@ "Connecting...": "", "Cookie policy": "", "Copied to clipboard": "", - "Copy and close": "", + "Copied!": "", + "Copy Quote": "", + "Copy caption for Instagram/TikTok": "", "Could not connect to the server": "", "Could not load the user details": "", "Could not load the user role": "", @@ -61,7 +66,6 @@ "Create account": "", "Created at": "", "Current password": "", - "Dark": "", "Dashboard": "", "Date": "", "December": "", @@ -79,7 +83,8 @@ "Disable 2FA authentication": "", "Disable account": "", "Disable authentication": "", - "Don't have an account yet? Sign up": "", + "Don’t have an account yet? Sign up": "", + "Duration": "", "Edit": "", "Edit profile": "", "Edit role": "", @@ -87,13 +92,11 @@ "Email verification state": "", "Enable 2FA authentication": "", "Enable account": "", - "Enter a topic and watch two AI agents argue with real sources": "", - "Enter a topic...": "", "Enter a secret key": "", - "Enter an existing key": "", "Enter your code": "", "Enter your password": "", "Enter your username": "", + "Ethereal connection stable... distilling further archival fragments...": "", "Ex.: John Steward": "", "Ex.: Moderator": "", "Ex.: New York, NY": "", @@ -101,12 +104,13 @@ "Ex.: john.steward@mail.com": "", "Ex.: jsteward": "", "Ex.: moderator": "", + "Exchanges": "", + "Exit": "", + "Export": "", "Failed to copy to clipboard": "", "February": "", "Filters": "", - "For": "", "Forgot your password?": "", - "From": "", "General": "", "Generate secure password": "", "Go back": "", @@ -118,6 +122,7 @@ "Grants the ability to create, edit, and delete roles, and manage the permissions assigned to them.": "", "Grants the ability to list and moderate users on the platform.": "", "Grants the permission to ban users from the platform.": "", + "Have real voice conversations with history's greatest minds.": "", "Home": "", "ID": "", "IP Address": "", @@ -126,15 +131,13 @@ "Include numbers and special characters for extra security.": "", "Incorrect password": "", "Internal server error": "", + "Interrupt": "", "Invalid URL.": "", - "Is AI consciousness possible?": "", - "Is social media good for society?": "", "Invalid email address.": "", "Invalid email provided": "", "Invalid file structure": "", "Invalid image provided": "", "Invalid location": "", - "Invalid name provided": "", "Invalid password provided": "", "Invalid profile description": "", "Invalid profile name": "", @@ -149,9 +152,8 @@ "June": "", "Keep your TOTP secret key stored in a safe location in your devices before proceeding with the activation.": "", "Language": "", - "Light": "", - "Live": "", - "Live debate": "", + "LinkedIn": "", + "Listening...": "", "Loading": "", "Location": "", "Login": "", @@ -161,12 +163,13 @@ "Manage user": "", "Manage users": "", "Management": "", + "Manifestation of": "", "March": "", "May": "", + "Metadata": "", "Moderate users": "", "Moderation inmune": "", "Moderator": "", - "N/A": "", "Name": "", "New password": "", "New permission": "", @@ -180,7 +183,6 @@ "No results found": "", "No roles available": "", "No sessions available": "", - "No sources yet": "", "No users available": "", "Not banned": "", "Not defined": "", @@ -206,7 +208,6 @@ "Oops! An error happened when loading users!": "", "Operative system": "", "Or continue with": "", - "Overview": "", "Page not found": "", "Password": "", "Password is insecure. Please fulfill all conditions.": "", @@ -220,27 +221,30 @@ "Past week": "", "Past year": "", "Permissions": "", - "Pineapple on pizza": "", "Please add permissions before adding a role.": "", "Please enter your password to confirm the change": "", "Please fill all required fields": "", "Please select an option.": "", "Please try again later or contact support": "", "Please, prove you are not a robot by solving the captcha.": "", + "Post to X": "", "Privacy policy": "", "Processing": "", "Profile data": "", "Provides immunity from moderation actions.": "", + "Ready": "", "Recover": "", "Recover password": "", "Remote device session removed": "", - "Remote work vs office": "", "Remove": "", "Repeat new password": "", "Repeat password": "", "Request was cancelled": "", "Reset password": "", "Restore filters": "", + "Resume": "", + "Resume conversation?": "", + "Resurrect the voices of the past through high-fidelity neural emulation. A portal to the Ethereal Archive.": "", "Retry in": "", "Retry limit reached. Please refresh the page or contact support.": "", "Role": "", @@ -257,33 +261,43 @@ "Saving changes": "", "Scan this QR code with your phone": "", "Search by id, username or email": "", - "Secret key": "", "Secret key copied to clipboard": "", "Secure password generated. Copied to clipboard": "", "Security": "", + "Select Your Guide": "", "Select a language": "", "Select a theme": "", "September": "", - "Session ended": "", + "Session Archived": "", + "Session Ended": "", "Settings": "", - "Show key": "", + "Share": "", + "Share on LinkedIn": "", + "Share on X": "", "Showing": "", "Sign up": "", "Sign up with Google": "", "Single-use code": "", "Something went wrong on our end. Please try again later.": "", + "Sources": "", + "Sources Found": "", + "Speaking...": "", "Start date must be before or equal to end date": "", - "Start debate": "", - "Stop debate": "", + "Start fresh": "", + "State": "", "Submitting": "", - "System": "", + "Summoned from": "", + "Summoning spirits...": "", + "Switch to auto": "", + "Switch to push-to-talk": "", "TFA Login": "", - "Tabs vs spaces": "", "TOTP Secret": "", "Tell us a bit about yourself. Ex.: Passionate about tech, design, and coffee.": "", "Terms & Conditions": "", + "The Digital Séance is Now Open": "", + "The Final Exchange": "", + "The Final Transcription": "", "The credentials you entered are not correct.": "", - "The current password is incorrect": "", "The page you are looking for does not exist or may have been moved. Please try with another page.": "", "The provided email is already in use": "", "The provided password is too short": "", @@ -294,10 +308,10 @@ "The user account was successfully disabled": "", "The user account was successfully reactivated": "", "The user you are trying to log into is banned from the platform": "", - "Theme": "", "There are currently no permissions to display.": "", "There are currently no roles to display.": "", "There are currently no user active sessions to display.": "", + "There are currently no users to display.": "", "This field is required.": "", "This permission group has no permissions assigned to it.": "", "This permission group has no permissions assigned to it. Add some permissions.": "", @@ -305,7 +319,6 @@ "This user has two-factor authentification enabled": "", "This user is immune to moderation": "", "This verification link is either invalid or has expired": "", - "To": "", "Try adjusting your filtering settings": "", "Try again": "", "Two factor authentication is already enabled for this account": "", @@ -313,7 +326,6 @@ "Two-factor authentication": "", "Two-factor authentication disabled": "", "Two-factor authentication enabled": "", - "Type": "", "Upload a file or drag and drop": "", "Upload new file": "", "Uploading image...": "", @@ -335,38 +347,25 @@ "Users": "", "Users request exceeded the allowed time limit.": "", "Verified": "", + "Verified Artifact": "", "View details": "", "We did not find any account with the provided email": "", "Website": "", - "What should they debate?": "", - "We've sent a recovery link to your email address. Follow the instructions to reset your password.": "", + "We’ve sent a recovery link to your email address. Follow the instructions to reset your password.": "", "Yes": "", + "You": "", "You are not authorized to view this profile": "", "You cannot change your own role": "", + "You have a previous conversation with": "", "You lack the required permission to access this resource": "", "Your language and theme preferences are loaded from your saved user settings. You can change them from the header at any time, but they will revert to your saved settings on page reload.": "", - "You've created a new password successfully. You can now log in.": "", + "You’ve created a new password successfully. You can now log in.": "", "[m] [d], [y]": "", "[m] [d], [y] [hh]:[mm]:[ss]": "", "current": "", - "default": "", "of": "", "result": "", "results": "", "to": "", - "up to": "", - "You:": "", - "Sources will appear here when the agent cites them...": "", - "Listening...": "", - "Speaking...": "", - "Ready": "", - "Auto": "", - "Push": "", - "End": "", - "Back": "", - "DeadTalk": "", - "DeadTalk - Talk to History": "", - "Have real voice conversations with historical figures, powered by real web sources.": "", - "Loading personas...": "", - "Built with ElevenLabs + Firecrawl for #ElevenHacks": "" + "up to": "" } diff --git a/web-application/frontend/i18n/locales/es.json b/web-application/frontend/i18n/locales/es.json index 29e1a3c..6b3175c 100644 --- a/web-application/frontend/i18n/locales/es.json +++ b/web-application/frontend/i18n/locales/es.json @@ -7,10 +7,10 @@ "Access denied. You lack the required permission to visit this page.": "Acceso denegado. No tienes el permiso requerido para visitar esta página.", "Account": "Cuenta", "Account created": "Cuenta creada", + "Account info": "Información de la cuenta", "Account settings": "Configuración cuenta", "Action denied. You lack the required permission to perform this action.": "Acción denegada. No tienes el permiso requerido para realizar esta acción.", "Active sessions": "Sesiones activas", - "Against": "En contra", "Admin": "Administrador", "Admin panel": "Panel de admin", "All banned status": "Todos los estados bloqueados", @@ -20,6 +20,8 @@ "An unknown error occurred": "Ocurrió un error desconocido", "Application": "Aplicación", "April": "Abril", + "Archival References": "Referencias de archivo", + "Archive Verified": "Verificado por el archivo", "Are you sure you want to close all sessions?": "¿Estás seguro de que quieres cerrar todas las sesiones?", "Are you sure you want to close this session?": "¿Estás seguro de que quieres cerrar esta sesión?", "Are you sure you want to delete this role?": "¿Estás seguro de que quieres eliminar este rol?", @@ -31,19 +33,20 @@ "August": "Agosto", "Avatar": "Avatar", "Avatar removed successfully": "Avatar eliminado correctamente", - "Back to home": "Volver al inicio", "Back to home page": "Volver a la página principal", "Back to homepage": "Volver a inicio", "Ban users": "Bloquear usuarios", "Banned": "Expulsado", + "Begin the Séance": "Comienza la sesión", "Both passwords must match.": "Ambas contraseñas deben coincidir.", "Browser": "Navegador", + "Built with ElevenLabs + Firecrawl for #ElevenHacks": "Hecho con ElevenLabs + Firecrawl para #ElevenHacks", "Cancel": "Cancelar", + "Change account password": "Cambiar la contraseña de la cuenta", "Change password": "Cambiar contraseña", "Changes saved": "Cambios guardados", - "Check secret key": "Comprobar clave secreta", "Check your inbox for an email we have sent to your new email in order to verify it and complete the change.": "Revisa tu bandeja de entrada, te hemos enviado un correo a tu nuevo email para verificarlo y completar el cambio", - "Checking": "Comprobando", + "Choose a consciousness to tether": "Elige una conciencia a la que vincularte", "Close": "Cerrar", "Close all sesions": "Cerrar todas las sesiones", "Close all session": "Cerrar todas las sesiones", @@ -52,7 +55,9 @@ "Connecting...": "Conectando...", "Cookie policy": "Política de cookies", "Copied to clipboard": "Copiado al portapapeles", - "Copy and close": "Copiar y cerrar", + "Copied!": "¡Copiado!", + "Copy Quote": "Copiar cita", + "Copy caption for Instagram/TikTok": "Copiar texto para Instagram/TikTok", "Could not connect to the server": "No se pudo conectar al servidor", "Could not load the user details": "No se pudieron cargar los detalles del usuario.", "Could not load the user role": "No se pudo cargar el rol del usuario.", @@ -61,7 +66,6 @@ "Create account": "Crear cuenta", "Created at": "Fecha de creación", "Current password": "Contraseña actual", - "Dark": "Oscuro", "Dashboard": "Escritorio", "Date": "Fecha", "December": "Diciembre", @@ -79,7 +83,8 @@ "Disable 2FA authentication": "Desactivar autenticación 2FA", "Disable account": "Deshabilitar cuenta", "Disable authentication": "Desactivar autenticación", - "Don't have an account yet? Sign up": "¿Aún no tienes una cuenta? Regístrate", + "Don’t have an account yet? Sign up": "¿Aún no tienes cuenta? Regístrate", + "Duration": "Duración", "Edit": "Editar", "Edit profile": "Editar perfil", "Edit role": "Editar rol", @@ -88,12 +93,10 @@ "Enable 2FA authentication": "Habilitar autenticación de 2FA", "Enable account": "Habilitar cuenta", "Enter a secret key": "Introduce una clave secreta", - "Enter a topic and watch two AI agents argue with real sources": "Introduce un tema y observa a dos agentes IA debatir con fuentes reales", - "Enter a topic...": "Introduce un tema...", - "Enter an existing key": "Introduce una clave existente", "Enter your code": "Introduce tu código", "Enter your password": "Introduce tu contraseña", "Enter your username": "Introduce tu nombre de usuario", + "Ethereal connection stable... distilling further archival fragments...": "La conexión etérea es estable... destilando más fragmentos de archivo...", "Ex.: John Steward": "Ej.: John Steward", "Ex.: Moderator": "Ej.: Moderador", "Ex.: New York, NY": "Ej.: Nueva York, NY", @@ -101,12 +104,13 @@ "Ex.: john.steward@mail.com": "Ej.: john.stewardmail.com", "Ex.: jsteward": "Ej.: jsteward", "Ex.: moderator": "Ej.: moderador ", + "Exchanges": "Intercambios", + "Exit": "Salir", + "Export": "Exportar", "Failed to copy to clipboard": "No se pudo copiar al portapapeles", "February": "Febrero", "Filters": "Filtros", - "For": "A favor", "Forgot your password?": "¿Olvidaste tu contraseña?", - "From": "Desde", "General": "General", "Generate secure password": "Generar contraseña segura", "Go back": "Volver atrás", @@ -118,6 +122,7 @@ "Grants the ability to create, edit, and delete roles, and manage the permissions assigned to them.": "Concede la capacidad para crear, editar y eliminar roles, y gestionar los permisos asignados a ellos.", "Grants the ability to list and moderate users on the platform.": "Concede la capacidad para listar y moderar usuarios en la plataforma.", "Grants the permission to ban users from the platform.": "Concede el permiso para bloquear usuarios de la plataforma.", + "Have real voice conversations with history's greatest minds.": "Mantén conversaciones de voz reales con las mentes más brillantes de la historia.", "Home": "Inicio", "ID": "ID", "IP Address": "Dirección IP", @@ -126,13 +131,13 @@ "Include numbers and special characters for extra security.": "Incluye números y caracteres especiales para mayor seguridad.", "Incorrect password": "Contraseña incorrecta", "Internal server error": "Error interno del servidor", + "Interrupt": "Interrumpir", "Invalid URL.": "URL inválida.", "Invalid email address.": "Correo electrónico inválido.", "Invalid email provided": "Correo electrónico no válido", "Invalid file structure": "Estructura de archivo no válida", "Invalid image provided": "Imagen no válida", "Invalid location": "Ubicación no válida", - "Invalid name provided": "Nombre no válido", "Invalid password provided": "Contraseña no válida", "Invalid profile description": "Descripción de perfil no válida", "Invalid profile name": "Nombre de perfil no válido", @@ -142,16 +147,13 @@ "Invalid single-use code": "Código de un solo uso no válido", "Invalid username provided": "Nombre de usuario no válido", "Invalid website. Must be a valid URL.": "Sitio web no válido. Debe ser una URL válida", - "Is AI consciousness possible?": "¿Es posible la consciencia de la IA?", - "Is social media good for society?": "¿Las redes sociales son buenas para la sociedad?", "January": "Enero", "July": "Julio", "June": "Junio", "Keep your TOTP secret key stored in a safe location in your devices before proceeding with the activation.": "Mantén tu clave secreta TOTP almacenada en una ubicación segura en tus dispositivos antes de continuar con la activación.", "Language": "Idioma", - "Light": "Claro", - "Live": "En vivo", - "Live debate": "Debate en vivo", + "LinkedIn": "LinkedIn", + "Listening...": "Escuchando...", "Loading": "Cargando", "Location": "Ubicación", "Login": "Iniciar sesión", @@ -161,12 +163,13 @@ "Manage user": "Gestionar usuario", "Manage users": "Gestionar usuarios", "Management": "Gestión", + "Manifestation of": "Manifestación de", "March": "Marzo", "May": "Mayo", + "Metadata": "Metadatos", "Moderate users": "Moderar usuarios", "Moderation inmune": "Inmune a la moderación", "Moderator": "Moderador", - "N/A": "N/A", "Name": "Nombre", "New password": "Nueva contraseña", "New permission": "Nuevo permiso", @@ -180,7 +183,6 @@ "No results found": "No se encontraron resultados", "No roles available": "No hay roles disponibles", "No sessions available": "No hay sesiones disponibles", - "No sources yet": "Sin fuentes aún", "No users available": "No hay usuarios disponibles", "Not banned": "No bloqueado", "Not defined": "No definido", @@ -206,7 +208,6 @@ "Oops! An error happened when loading users!": "¡Vaya! Ocurrió un error al cargar los usuarios.", "Operative system": "Sistema operativo", "Or continue with": "O continuar con", - "Overview": "Resumen", "Page not found": "Página no encontrada", "Password": "Contraseña", "Password is insecure. Please fulfill all conditions.": "La contraseña es insegura. Por favor, cumple todas las condiciones.", @@ -220,27 +221,30 @@ "Past week": "Última semana", "Past year": "Último año", "Permissions": "Permisos", - "Pineapple on pizza": "Piña en la pizza", "Please add permissions before adding a role.": "Por favor, añade permisos antes de añadir un rol.", "Please enter your password to confirm the change": "Por favor, introduce tu contraseña para confirmar el cambio", "Please fill all required fields": "Por favor, completa todos los campos obligatorios", "Please select an option.": "Por favor, selecciona una opción.", "Please try again later or contact support": "Por favor, inténtalo de nuevo más tarde o contacta con soporte", "Please, prove you are not a robot by solving the captcha.": "Por favor, demuestra que no eres un robot resolviendo el captcha.", + "Post to X": "Publicar en X", "Privacy policy": "Política de privacidad", "Processing": "Procesando", "Profile data": "Datos del perfil", "Provides immunity from moderation actions.": "Proporciona inmunidad contra acciones de moderación.", + "Ready": "Listo", "Recover": "Recuperar", "Recover password": "Recuperar contraseña", "Remote device session removed": "Sesión de dispositivo remoto eliminada", - "Remote work vs office": "Teletrabajo vs oficina", "Remove": "Eliminar", "Repeat new password": "Repetir nueva contraseña", "Repeat password": "Repetir contraseña", "Request was cancelled": "La solicitud fue cancelada", "Reset password": "Restablecer contraseña", "Restore filters": "Restaurar filtros", + "Resume": "Reanudar", + "Resume conversation?": "¿Reanudar conversación?", + "Resurrect the voices of the past through high-fidelity neural emulation. A portal to the Ethereal Archive.": "Resucita las voces del pasado mediante una emulación neuronal de alta fidelidad. Un portal al Archivo Etéreo.", "Retry in": "Reintentar en", "Retry limit reached. Please refresh the page or contact support.": "Límite de reintentos alcanzado. Por favor, actualiza la página o contacta con soporte.", "Role": "Rol", @@ -257,33 +261,43 @@ "Saving changes": "Guardando cambios", "Scan this QR code with your phone": "Escanea este código QR con tu teléfono", "Search by id, username or email": "Buscar por id, nombre de usuario o correo electrónico", - "Secret key": "Clave secreta", "Secret key copied to clipboard": "Clave secreta copiada al portapapeles", "Secure password generated. Copied to clipboard": "Contraseña segura generada. Copiada al portapapeles", "Security": "Seguridad", + "Select Your Guide": "Elige tu guía", "Select a language": "Selecciona un idioma", "Select a theme": "Selecciona un tema", "September": "Septiembre", - "Session ended": "Sesión finalizada", + "Session Archived": "Sesión archivada", + "Session Ended": "Sesión finalizada", "Settings": "Configuraciones", - "Show key": "Mostrar clave", + "Share": "Compartir", + "Share on LinkedIn": "Compartir en LinkedIn", + "Share on X": "Compartir en X", "Showing": "Mostrando", "Sign up": "Registrarse", "Sign up with Google": "Registrarse con Google", "Single-use code": "Código de un solo uso", "Something went wrong on our end. Please try again later.": "Algo salió mal de nuestro lado. Por favor, inténtalo de nuevo más tarde.", + "Sources": "Fuentes", + "Sources Found": "Fuentes encontradas", + "Speaking...": "Hablando...", "Start date must be before or equal to end date": "La fecha de inicio debe ser anterior o igual a la fecha de finalización", - "Start debate": "Iniciar debate", - "Stop debate": "Detener debate", + "Start fresh": "Empezar de cero", + "State": "Estado", "Submitting": "Enviando", - "System": "Sistema", + "Summoned from": "Invocado desde", + "Summoning spirits...": "Invocando espíritus...", + "Switch to auto": "Cambiar a automático", + "Switch to push-to-talk": "Cambiar a pulsar para hablar", "TFA Login": "Inicio de sesión con TFA.", "TOTP Secret": "Secreto TOTP", - "Tabs vs spaces": "Tabulaciones vs espacios", "Tell us a bit about yourself. Ex.: Passionate about tech, design, and coffee.": "Cuéntanos un poco sobre ti. Ej.: Apasionado por la tecnología, el diseño y el café.", "Terms & Conditions": "Términos y condiciones", + "The Digital Séance is Now Open": "La sesión digital ya está abierta", + "The Final Exchange": "El último intercambio", + "The Final Transcription": "La transcripción final", "The credentials you entered are not correct.": "Las credenciales que ingresaste no son correctas.", - "The current password is incorrect": "La contraseña actual es incorrecta", "The page you are looking for does not exist or may have been moved. Please try with another page.": "La página que buscas no existe o puede haber sido movida. Por favor, intenta con otra página.", "The provided email is already in use": "El correo electrónico proporcionado ya está en uso", "The provided password is too short": "La contraseña proporcionada es demasiado corta", @@ -294,10 +308,10 @@ "The user account was successfully disabled": "La cuenta del usuario se desactivó correctamente", "The user account was successfully reactivated": "La cuenta del usuario se reactivó correctamente", "The user you are trying to log into is banned from the platform": "El usuario al que intentas acceder está bloqueado en la plataforma", - "Theme": "Tema", "There are currently no permissions to display.": "Actualmente no hay permisos para mostrar. ", "There are currently no roles to display.": "Actualmente no hay roles para mostrar.", "There are currently no user active sessions to display.": "Actualmente no hay sesiones de usuario activas para mostrar.", + "There are currently no users to display.": "Actualmente no hay usuarios para mostrar.", "This field is required.": "Este campo es obligatorio.", "This permission group has no permissions assigned to it.": "Este grupo de permisos no tiene permisos asignados.", "This permission group has no permissions assigned to it. Add some permissions.": "Este grupo de permisos no tiene permisos asignados. Añade algunos permisos.", @@ -305,7 +319,6 @@ "This user has two-factor authentification enabled": "Este usuario tiene habilitada la autenticación en dos factores", "This user is immune to moderation": "Este usuario es inmune a la moderación", "This verification link is either invalid or has expired": "Este enlace de verificación es inválido o ha expirado", - "To": "Hasta", "Try adjusting your filtering settings": "Intenta ajustar tu configuración de filtros", "Try again": "Intenta de nuevo", "Two factor authentication is already enabled for this account": "La autenticación de dos factores ya está activada para esta cuenta", @@ -313,7 +326,6 @@ "Two-factor authentication": "Autenticación de dos factores", "Two-factor authentication disabled": "Autenticación de dos factores deshabilitada", "Two-factor authentication enabled": "Autenticación de dos factores habilitada", - "Type": "Tipo", "Upload a file or drag and drop": "Sube un archivo o arrástralo y suéltalo", "Upload new file": "Subir nuevo archivo", "Uploading image...": "Subiendo imagen...", @@ -335,38 +347,25 @@ "Users": "Usuarios", "Users request exceeded the allowed time limit.": "La solicitud de usuarios superó el límite de tiempo permitido.", "Verified": "Verificado", + "Verified Artifact": "Artefacto verificado", "View details": "Ver detalles", "We did not find any account with the provided email": "No encontramos ninguna cuenta con el correo proporcionado", "Website": "Sitio web", - "We've sent a recovery link to your email address. Follow the instructions to reset your password.": "Hemos enviado un enlace de recuperación a tu dirección de correo electrónico. Sigue las instrucciones para restablecer tu contraseña.", - "What should they debate?": "¿Sobre qué deberían debatir?", + "We’ve sent a recovery link to your email address. Follow the instructions to reset your password.": "Hemos enviado un enlace de recuperación a tu correo electrónico. Sigue las instrucciones para restablecer tu contraseña.", "Yes": "Sí", + "You": "Tú", "You are not authorized to view this profile": "No estás autorizado para ver este perfil", "You cannot change your own role": "No puedes cambiar tu propio rol.", + "You have a previous conversation with": "Tienes una conversación anterior con", "You lack the required permission to access this resource": "No tienes el permiso necesario para acceder a este recurso", "Your language and theme preferences are loaded from your saved user settings. You can change them from the header at any time, but they will revert to your saved settings on page reload.": "Tus preferencias de idioma y tema se cargan desde tu configuración de usuario guardada. Puedes cambiarlas desde el encabezado en cualquier momento, pero volverán a tu configuración guardada al recargar la página.", - "You've created a new password successfully. You can now log in.": "Has creado una nueva contraseña con éxito. Ahora puedes iniciar sesión.", + "You’ve created a new password successfully. You can now log in.": "Has creado una nueva contraseña correctamente. Ya puedes iniciar sesión.", "[m] [d], [y]": "[d] de [m], [y]", "[m] [d], [y] [hh]:[mm]:[ss]": "[d] de [m], [y] [hh]:[mm]:[ss]", "current": "actual", - "default": "predeterminado", "of": "de", "result": "resultado", "results": "resultados", "to": "a", - "up to": "hasta", - "You:": "Tú:", - "Sources will appear here when the agent cites them...": "Las fuentes aparecerán aquí cuando el agente las cite...", - "Listening...": "Escuchando...", - "Speaking...": "Hablando...", - "Ready": "Listo", - "Auto": "Auto", - "Push": "Pulsar", - "End": "Fin", - "Back": "Volver", - "DeadTalk": "DeadTalk", - "DeadTalk - Talk to History": "DeadTalk - Habla con la Historia", - "Have real voice conversations with historical figures, powered by real web sources.": "Conversa con figuras históricas mediante voz real, con fuentes web verificadas.", - "Loading personas...": "Cargando personajes...", - "Built with ElevenLabs + Firecrawl for #ElevenHacks": "Hecho con ElevenLabs + Firecrawl para #ElevenHacks" + "up to": "hasta" } diff --git a/web-application/frontend/layouts/seance.vue b/web-application/frontend/layouts/seance.vue new file mode 100644 index 0000000..812c2c7 --- /dev/null +++ b/web-application/frontend/layouts/seance.vue @@ -0,0 +1,22 @@ + + + diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index 521709b..29ba0b0 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -1,65 +1,157 @@ From 35b8bb53e5ce7628fb02ac9902f8b00fc8d0265e Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sun, 22 Mar 2026 13:33:35 +0100 Subject: [PATCH 11/18] fix(copilot): suggestions --- .../backend/src/controllers/api/api-session.ts | 17 ++++++++++++++--- .../frontend/components/seance/SeanceHeader.vue | 12 +----------- .../components/session/SessionControls.vue | 4 ---- .../session/SessionTranscriptOverlay.vue | 4 ---- .../frontend/composables/useVoiceInput.ts | 4 ++-- web-application/frontend/i18n/locales/en.json | 2 ++ web-application/frontend/i18n/locales/es.json | 2 ++ 7 files changed, 21 insertions(+), 24 deletions(-) diff --git a/web-application/backend/src/controllers/api/api-session.ts b/web-application/backend/src/controllers/api/api-session.ts index d065909..48966d6 100644 --- a/web-application/backend/src/controllers/api/api-session.ts +++ b/web-application/backend/src/controllers/api/api-session.ts @@ -45,19 +45,30 @@ export class SessionController extends Controller { * Generates a memorable quote from the conversation transcript using LLM. * Takes the full conversation and distills it into a single evocative line * that captures the essence of the exchange with the historical figure. - * Binding: GenerateQuoteRequest + * Binding: GenerateQuote * @route POST /session/generate-quote * @group session * @param {GenerateQuoteRequest.model} request.body.required - Transcript and persona info * @returns {GenerateQuoteResponse.model} 200 - Generated quote * @returns {GenerateQuoteError.model} 400 - Invalid input + * @returns {GenerateQuoteError.model} 500 - Generation failed (GENERATION_FAILED) */ - private async generateQuote(req: Express.Request, res: Express.Response) { + public async generateQuote(req: Express.Request, res: Express.Response) { const { personaName, transcript } = req.body; - if (!personaName || !transcript || !Array.isArray(transcript) || transcript.length === 0) { + if (!personaName || typeof personaName !== "string") { return sendApiError(req, res, 400, "INVALID_INPUT"); } + if (!transcript || !Array.isArray(transcript) || transcript.length === 0) { + return sendApiError(req, res, 400, "INVALID_INPUT"); + } + + // Validate each transcript entry has required shape + for (const entry of transcript) { + if (!entry || typeof entry.role !== "string" || typeof entry.text !== "string") { + return sendApiError(req, res, 400, "INVALID_INPUT"); + } + } try { const openai = OpenAIService.getInstance(); diff --git a/web-application/frontend/components/seance/SeanceHeader.vue b/web-application/frontend/components/seance/SeanceHeader.vue index 6fcd90d..da71394 100644 --- a/web-application/frontend/components/seance/SeanceHeader.vue +++ b/web-application/frontend/components/seance/SeanceHeader.vue @@ -26,17 +26,7 @@ {{ $t("Archive Verified") }} - + diff --git a/web-application/frontend/components/session/SessionControls.vue b/web-application/frontend/components/session/SessionControls.vue index 3ee9d93..40212ed 100644 --- a/web-application/frontend/components/session/SessionControls.vue +++ b/web-application/frontend/components/session/SessionControls.vue @@ -217,20 +217,16 @@ const statusLabelClass = computed(() => { let pttActive = false; function onPttDown() { - console.log("[PTT] pointerdown fired, pttActive:", pttActive); if (pttActive) return; pttActive = true; - console.log("[PTT] emitting push-start"); emit("push-start"); window.addEventListener("pointerup", onPttUp, { once: true }); window.addEventListener("pointercancel", onPttUp, { once: true }); } function onPttUp() { - console.log("[PTT] pointerup/cancel fired, pttActive:", pttActive); if (!pttActive) return; pttActive = false; - console.log("[PTT] emitting push-end"); emit("push-end"); window.removeEventListener("pointerup", onPttUp); window.removeEventListener("pointercancel", onPttUp); diff --git a/web-application/frontend/components/session/SessionTranscriptOverlay.vue b/web-application/frontend/components/session/SessionTranscriptOverlay.vue index 614d318..6a22351 100644 --- a/web-application/frontend/components/session/SessionTranscriptOverlay.vue +++ b/web-application/frontend/components/session/SessionTranscriptOverlay.vue @@ -30,10 +30,6 @@ const props = defineProps({ type: String as PropType, default: "", }, - personaName: { - type: String as PropType, - default: "", - }, }); const isAgent = computed(() => !!props.agentTranscript); diff --git a/web-application/frontend/composables/useVoiceInput.ts b/web-application/frontend/composables/useVoiceInput.ts index 3be7989..00bbade 100644 --- a/web-application/frontend/composables/useVoiceInput.ts +++ b/web-application/frontend/composables/useVoiceInput.ts @@ -182,8 +182,8 @@ export function useVoiceInput(sendMessage: (msg: any) => void, isAgentPlaying?: pttWaitingForSpeechEnd = false; isSpeaking.value = true; - // Ensure VAD is running - if (vadInstance) { + // Ensure VAD is running, but not while agent is playing (echo suppression) + if (vadInstance && !isAgentPlaying?.value) { try { vadInstance.start(); } catch { diff --git a/web-application/frontend/i18n/locales/en.json b/web-application/frontend/i18n/locales/en.json index 209d605..9bd2f70 100644 --- a/web-application/frontend/i18n/locales/en.json +++ b/web-application/frontend/i18n/locales/en.json @@ -59,6 +59,7 @@ "Copy Quote": "", "Copy caption for Instagram/TikTok": "", "Could not connect to the server": "", + "DeadTalk - Talk to History": "", "Could not load the user details": "", "Could not load the user role": "", "Could not load user activation state because the user does not exist.": "", @@ -234,6 +235,7 @@ "Provides immunity from moderation actions.": "", "Ready": "", "Recover": "", + "{count} Spirits Summoned Today": "{count} Spirits Summoned Today", "Recover password": "", "Remote device session removed": "", "Remove": "", diff --git a/web-application/frontend/i18n/locales/es.json b/web-application/frontend/i18n/locales/es.json index 6b3175c..a18f145 100644 --- a/web-application/frontend/i18n/locales/es.json +++ b/web-application/frontend/i18n/locales/es.json @@ -59,6 +59,7 @@ "Copy Quote": "Copiar cita", "Copy caption for Instagram/TikTok": "Copiar texto para Instagram/TikTok", "Could not connect to the server": "No se pudo conectar al servidor", + "DeadTalk - Talk to History": "DeadTalk - Habla con la Historia", "Could not load the user details": "No se pudieron cargar los detalles del usuario.", "Could not load the user role": "No se pudo cargar el rol del usuario.", "Could not load user activation state because the user does not exist.": "No se pudo cargar el estado de activación del usuario porque no existe", @@ -234,6 +235,7 @@ "Provides immunity from moderation actions.": "Proporciona inmunidad contra acciones de moderación.", "Ready": "Listo", "Recover": "Recuperar", + "{count} Spirits Summoned Today": "{count} espíritus invocados hoy", "Recover password": "Recuperar contraseña", "Remote device session removed": "Sesión de dispositivo remoto eliminada", "Remove": "Eliminar", From 8f655d1404c4c88d6315b116c2fad79a56a8641c Mon Sep 17 00:00:00 2001 From: Diego Valdeolmillos Date: Sun, 22 Mar 2026 21:00:30 +0100 Subject: [PATCH 12/18] feat: Custom Character Creation & Conversation Engine Improvements --- .../src/config/config-image-generation.ts | 45 + .../backend/src/config/personas.ts | 16 +- .../src/controllers/api/api-characters.ts | 136 +++ .../src/controllers/api/api-personas.ts | 78 +- .../src/controllers/websocket/websocket.ts | 400 +++---- .../backend/src/models/character-research.ts | 70 ++ .../backend/src/models/custom-persona.ts | 138 +++ .../services/character-creation-service.ts | 380 ++++++ .../services/character-research-service.ts | 243 ++++ .../services/conversation-engine-service.ts | 1022 +++++++++-------- .../src/services/elevenlabs-service.ts | 165 +++ .../src/services/image-generation-service.ts | 182 +++ .../src/services/voice-discovery-service.ts | 198 ++++ .../src/services/ws-orchestrator-service.ts | 5 +- .../components/session/BackgroundEffect.vue | 364 ++++++ .../components/session/SessionPortrait.vue | 3 +- .../composables/useOrchestratorSocket.ts | 2 +- web-application/frontend/i18n/locales/en.json | 19 +- web-application/frontend/i18n/locales/es.json | 19 +- .../frontend/pages/create-character.vue | 331 ++++++ web-application/frontend/pages/index.vue | 89 +- web-application/frontend/pages/session.vue | 7 + .../public/images/personas/cleopatra.png | Bin 0 -> 2686479 bytes .../frontend/public/images/personas/curie.png | Bin 0 -> 3135354 bytes .../public/images/personas/einstein.png | Bin 0 -> 3306123 bytes .../frontend/public/images/personas/jobs.png | Bin 0 -> 2726778 bytes 26 files changed, 3196 insertions(+), 716 deletions(-) create mode 100644 web-application/backend/src/config/config-image-generation.ts create mode 100644 web-application/backend/src/controllers/api/api-characters.ts create mode 100644 web-application/backend/src/models/character-research.ts create mode 100644 web-application/backend/src/models/custom-persona.ts create mode 100644 web-application/backend/src/services/character-creation-service.ts create mode 100644 web-application/backend/src/services/character-research-service.ts create mode 100644 web-application/backend/src/services/image-generation-service.ts create mode 100644 web-application/backend/src/services/voice-discovery-service.ts create mode 100644 web-application/frontend/components/session/BackgroundEffect.vue create mode 100644 web-application/frontend/pages/create-character.vue create mode 100644 web-application/frontend/public/images/personas/cleopatra.png create mode 100644 web-application/frontend/public/images/personas/curie.png create mode 100644 web-application/frontend/public/images/personas/einstein.png create mode 100644 web-application/frontend/public/images/personas/jobs.png diff --git a/web-application/backend/src/config/config-image-generation.ts b/web-application/backend/src/config/config-image-generation.ts new file mode 100644 index 0000000..3a4dfb3 --- /dev/null +++ b/web-application/backend/src/config/config-image-generation.ts @@ -0,0 +1,45 @@ +// Image generation configuration (Google Imagen / DALL-E) + +"use strict"; + +/** + * Image generation configuration. + */ +export class ImageGenerationConfig { + + /** + * Gets the configuration instance. + */ + public static getInstance(): ImageGenerationConfig { + if (ImageGenerationConfig.instance) { + return ImageGenerationConfig.instance; + } + + const config: ImageGenerationConfig = new ImageGenerationConfig(); + + config.googleApiKey = process.env.GOOGLE_GENAI_API_KEY || ""; + config.imagenModel = process.env.IMAGEN_MODEL || "imagen-4.0-generate-001"; + config.openaiApiKey = process.env.OPENAI_API_KEY || ""; + config.provider = config.googleApiKey ? "imagen" : config.openaiApiKey ? "dalle" : "none"; + config.enabled = config.provider !== "none"; + + ImageGenerationConfig.instance = config; + + return config; + } + private static instance: ImageGenerationConfig = null; + + public googleApiKey: string; + public imagenModel: string; + public openaiApiKey: string; + public provider: "imagen" | "dalle" | "none"; + public enabled: boolean; + + constructor() { + this.googleApiKey = ""; + this.imagenModel = "imagen-4.0-generate-001"; + this.openaiApiKey = ""; + this.provider = "none"; + this.enabled = false; + } +} diff --git a/web-application/backend/src/config/personas.ts b/web-application/backend/src/config/personas.ts index c4d450c..bab00b5 100644 --- a/web-application/backend/src/config/personas.ts +++ b/web-application/backend/src/config/personas.ts @@ -263,8 +263,8 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "angry", trigger: "War, nationalism, racism, intolerance" }, { emotion: "whispers", trigger: "Failed marriages, personal regrets, son Eduard" }, ], - avatar: "/images/personas/einstein.jpg", - image: "https://lh3.googleusercontent.com/aida-public/AB6AXuD6Sg0caTpkA1Yz_yGd-bh2ri6EzC9_VMA4eA7qornPlfT1fcIngg96pVfc90Xxh4gNZNm3TFcwyueUvzCH5M1J_j7dxg80-mHR4gy0RRQtd1T6Sj6uzpR1GqHLlf639zJuXZNkgwNZAfZ5DCKwV_L4jpFkjQNXDTUnviaZcqTrdPWPHiXqQl23hGMxisj-_H1a9Pi42wTfI8w1AYH5O5YeSTFrflfTtMa-_W-bJEPmghjn7Fhq10VSdejjyxuqPkF3g9vJwSSt81_2", + avatar: "/images/personas/einstein.png", + image: "/images/personas/einstein.png", quote: "Imagination is more important than knowledge. For knowledge is limited, whereas imagination embraces the entire world.", quoteEs: "La imaginación es más importante que el conocimiento. Porque el conocimiento es limitado, mientras que la imaginación abarca el mundo entero.", searchKeywords: ["physics", "relativity", "quantum", "Nobel Prize", "pacifism"], @@ -361,8 +361,8 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "whispers", trigger: "Pierre's death, radiation sickness, loneliness" }, { emotion: "pause", trigger: "Langevin scandal, Academy rejection" }, ], - avatar: "/images/personas/curie.jpg", - image: "", + avatar: "/images/personas/curie.png", + image: "/images/personas/curie.png", quote: "Nothing in life is to be feared, it is only to be understood.", quoteEs: "Nada en la vida debe ser temido, solo debe ser comprendido.", searchKeywords: ["radioactivity", "radium", "polonium", "Nobel Prize", "women in science"], @@ -460,8 +460,8 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "whispers", trigger: "Caesar's death, Antony's flaws, her final choice" }, { emotion: "pause", trigger: "End of dynasty, fall of Egypt, children's fates" }, ], - avatar: "/images/personas/cleopatra.jpg", - image: "https://lh3.googleusercontent.com/aida-public/AB6AXuBlKFbzesyN3MQW8yCX8b20xZZmWSlxVkyhxnARrmeFjFgmhqWUJfM_tKcaz5gDLCG96vNyhQ4n0jpN4Cd4VTEWtHXxMW7Y_Le-LtqdUcj232S5ISwZnTndgTYIxhfYw5HDMTXyuNtWuQ1njuhs1GYoNEzqFXUP4TeDxM0Hyvs2B__ORHl1AtS3LkAeBKa3xtlAUEcJeZZYfrlQOk5HZL6PwvjjN8NdarGMmIWedJ_UQBLJMZMh3H8ljGeqa_EP1ybHAB0I0Guf2-4r", + avatar: "/images/personas/cleopatra.png", + image: "/images/personas/cleopatra.png", quote: "I will not be triumphed over.", quoteEs: "No seré exhibida en triunfo.", searchKeywords: ["pharaoh", "Egypt", "Rome", "Antony", "Caesar", "Ptolemaic"], @@ -560,8 +560,8 @@ ${SYSTEM_PROMPT_INSTRUCTIONS}`, { emotion: "whispers", trigger: "Adoption, daughter Lisa, cancer, mortality" }, { emotion: "pause", trigger: "Legacy, returning to Apple, making a dent" }, ], - avatar: "/images/personas/jobs.jpg", - image: "", + avatar: "/images/personas/jobs.png", + image: "/images/personas/jobs.png", quote: "Stay hungry, stay foolish.", quoteEs: "Sigue hambriento, sigue alocado.", searchKeywords: ["Apple", "iPhone", "design", "innovation", "Silicon Valley"], diff --git a/web-application/backend/src/controllers/api/api-characters.ts b/web-application/backend/src/controllers/api/api-characters.ts new file mode 100644 index 0000000..46100c1 --- /dev/null +++ b/web-application/backend/src/controllers/api/api-characters.ts @@ -0,0 +1,136 @@ +// Characters API controller — custom character creation + +"use strict"; + +import Express from "express"; +import { noCache, NOT_FOUND, sendApiError, sendApiResult } from "../../utils/http-utils"; +import { Controller } from "../controller"; +import { CharacterCreationService } from "../../services/character-creation-service"; +import { Monitor } from "../../monitor"; + +/** + * @typedef CharacterCreateBody + * @property {string} name.required - Historical figure name + * @property {string} hints - Optional hints (era, profession, etc.) + * @property {string} photoBase64 - Optional base64 portrait image + */ + +/** + * @typedef CharacterSummary + * @property {string} id - Persona slug identifier + * @property {string} name - Display name + * @property {string} era - Birth-death years + * @property {string} profession - Profession / title + * @property {string} image - Portrait image (data URL or empty) + * @property {string} voiceSource - How the voice was created (cloned, designed, default) + * @property {boolean} isCustom - Always true for custom characters + */ + +/** + * Characters API — endpoints for creating and managing custom historical characters. + * Binding: CharacterCreateBody + * @group characters - Custom Characters + */ +export class CharactersController extends Controller { + public registerAPI(prefix: string, application: Express.Express): any { + application.post(prefix + "/characters/create", noCache(this.createCharacter.bind(this))); + application.get(prefix + "/characters", noCache(this.listCharacters.bind(this))); + application.get(prefix + "/characters/:id", noCache(this.getCharacter.bind(this))); + } + + /** + * Creates a new custom historical character. + * Researches the figure via Firecrawl, generates portrait, creates voice. + * Binding: CharacterCreateBody + * @route POST /characters/create + * @group characters + * @param {CharacterCreateBody.model} request.body.required - Character creation request + * @returns {CharacterSummary.model} 200 - Created character + */ + private async createCharacter(request: Express.Request, response: Express.Response) { + const { name, hints, photoBase64 } = request.body || {}; + + if (!name || typeof name !== "string" || name.trim().length === 0) { + return sendApiError(request, response, 400, "INVALID_NAME", "Name is required"); + } + + try { + const persona = await CharacterCreationService.getInstance().createCharacter( + name.trim(), + hints?.trim(), + photoBase64, + ); + + sendApiResult(request, response, { + id: persona.id, + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + image: persona.image, + voiceSource: persona.voiceSource, + voiceId: persona.voiceId, + firstMessage: persona.firstMessage, + firstMessageEs: persona.firstMessageEs, + quote: persona.quote, + isCustom: true, + }); + } catch (err) { + Monitor.exception(err, "CharactersController.createCharacter failed"); + sendApiError(request, response, 500, "CREATE_FAILED", "Character creation failed: " + (err as Error).message); + } + } + + /** + * Lists all custom characters. + * @route GET /characters + * @group characters + * @returns {Array.} 200 - List of custom characters + */ + private async listCharacters(request: Express.Request, response: Express.Response) { + const personas = await CharacterCreationService.getInstance().listCustomPersonas(); + + sendApiResult(request, response, personas.map(p => ({ + id: p.id, + name: p.name, + era: p.era, + profession: p.profession, + image: p.image, + voiceSource: p.voiceSource, + isCustom: true, + }))); + } + + /** + * Gets a specific custom character by ID. + * @route GET /characters/:id + * @group characters + * @param {string} id.path.required - Character ID + * @returns {CharacterSummary.model} 200 - Character details + */ + private async getCharacter(request: Express.Request, response: Express.Response) { + const id = request.params.id; + const persona = await CharacterCreationService.getInstance().getCustomPersona(id); + + if (!persona) { + return sendApiError(request, response, NOT_FOUND, "CHARACTER_NOT_FOUND", "Character not found"); + } + + sendApiResult(request, response, { + id: persona.id, + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + image: persona.image, + voiceSource: persona.voiceSource, + voiceId: persona.voiceId, + firstMessage: persona.firstMessage, + firstMessageEs: persona.firstMessageEs, + quote: persona.quote, + emotionalProfile: persona.emotionalProfile, + searchKeywords: persona.searchKeywords, + isCustom: true, + }); + } +} diff --git a/web-application/backend/src/controllers/api/api-personas.ts b/web-application/backend/src/controllers/api/api-personas.ts index 82ea4cd..3a00b2c 100644 --- a/web-application/backend/src/controllers/api/api-personas.ts +++ b/web-application/backend/src/controllers/api/api-personas.ts @@ -6,6 +6,7 @@ import Express from "express"; import { noCache, NOT_FOUND, sendApiError, sendApiResult } from "../../utils/http-utils"; import { Controller } from "../controller"; import { PersonasConfig } from "../../config/personas"; +import { CharacterCreationService } from "../../services/character-creation-service"; /** * @typedef PersonaSummary @@ -69,8 +70,24 @@ export class PersonasController extends Controller { * @returns {Array.} 200 - List of personas */ public async listPersonas(request: Express.Request, response: Express.Response) { - const personas = PersonasConfig.getInstance().listPersonas(); - sendApiResult(request, response, personas); + const builtIn = PersonasConfig.getInstance().listPersonas(); + + // Append custom characters created via /characters/create + const custom = (await CharacterCreationService.getInstance().listCustomPersonas()).map(p => ({ + id: p.id, + name: p.name, + era: p.era, + nationality: p.nationality, + profession: p.profession, + avatar: p.avatar || "", + image: p.image, + quote: p.quote, + quoteEs: p.quoteEs, + firstMessage: p.firstMessage, + firstMessageEs: p.firstMessageEs, + })); + + sendApiResult(request, response, [...builtIn, ...custom]); } /** @@ -86,26 +103,47 @@ export class PersonasController extends Controller { const id = request.params.id || ""; const persona = PersonasConfig.getInstance().getPersonaById(id); - if (!persona) { - sendApiError(request, response, NOT_FOUND, "PERSONA_NOT_FOUND", "Persona not found: " + id); + if (persona) { + // Return public fields only (no system prompt) + sendApiResult(request, response, { + id: persona.id, + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + avatar: persona.avatar, + image: persona.image, + quote: persona.quote, + quoteEs: persona.quoteEs, + firstMessage: persona.firstMessage, + firstMessageEs: persona.firstMessageEs, + emotionalProfile: persona.emotionalProfile, + searchKeywords: persona.searchKeywords, + }); + return; + } + + // Check custom characters + const custom = await CharacterCreationService.getInstance().getCustomPersona(id); + if (custom) { + sendApiResult(request, response, { + id: custom.id, + name: custom.name, + era: custom.era, + nationality: custom.nationality, + profession: custom.profession, + avatar: custom.avatar || "", + image: custom.image, + quote: custom.quote, + quoteEs: custom.quoteEs, + firstMessage: custom.firstMessage, + firstMessageEs: custom.firstMessageEs, + emotionalProfile: custom.emotionalProfile, + searchKeywords: custom.searchKeywords, + }); return; } - // Return public fields only (no system prompt) - sendApiResult(request, response, { - id: persona.id, - name: persona.name, - era: persona.era, - nationality: persona.nationality, - profession: persona.profession, - avatar: persona.avatar, - image: persona.image, - quote: persona.quote, - quoteEs: persona.quoteEs, - firstMessage: persona.firstMessage, - firstMessageEs: persona.firstMessageEs, - emotionalProfile: persona.emotionalProfile, - searchKeywords: persona.searchKeywords, - }); + sendApiError(request, response, NOT_FOUND, "PERSONA_NOT_FOUND", "Persona not found: " + id); } } diff --git a/web-application/backend/src/controllers/websocket/websocket.ts b/web-application/backend/src/controllers/websocket/websocket.ts index 0781880..1113473 100644 --- a/web-application/backend/src/controllers/websocket/websocket.ts +++ b/web-application/backend/src/controllers/websocket/websocket.ts @@ -16,232 +16,232 @@ const WS_QUEUE_SIZE = 20; * Websocket controller */ export class WebsocketController { - public request: HTTP.IncomingMessage; - public socket: WebSocket.WebSocket; - public interval: any; - public busy: boolean; - public closed: boolean; - - public initT: number; - public lastAliveT: number; - - public queue: AsyncQueue; - - public logger: RequestLogger; - - public eventListeners: { [event: string]: (any) => any }; - - public sessionId: string | null; - - constructor(ws: WebSocket.WebSocket, req: HTTP.IncomingMessage) { - this.closed = false; - this.request = req; - this.socket = ws; - this.logger = new RequestLogger(); - this.busy = false; - this.interval = null; - this.queue = new AsyncQueue(WS_QUEUE_SIZE, this.parseMessage.bind(this)); - this.queue.on("error", (error) => { - Monitor.exception(error); - }); - this.sessionId = null; - } - - /* General */ - - public async tick() { - if (Date.now() - this.initT > 24 * 60 * 60 * 1000) { - Monitor.debug("Too much time connected, killing websocket"); - this.kill(); - return; // Too much time connected + public request: HTTP.IncomingMessage; + public socket: WebSocket.WebSocket; + public interval: any; + public busy: boolean; + public closed: boolean; + + public initT: number; + public lastAliveT: number; + + public queue: AsyncQueue; + + public logger: RequestLogger; + + public eventListeners: { [event: string]: (any) => any }; + + public sessionId: string | null; + + constructor(ws: WebSocket.WebSocket, req: HTTP.IncomingMessage) { + this.closed = false; + this.request = req; + this.socket = ws; + this.logger = new RequestLogger(); + this.busy = false; + this.interval = null; + this.queue = new AsyncQueue(WS_QUEUE_SIZE, this.parseMessage.bind(this)); + this.queue.on("error", (error) => { + Monitor.exception(error); + }); + this.sessionId = null; } - if (Date.now() - this.lastAliveT > 60 * 1000) { - Monitor.debug("Socket not alive"); - this.kill(); - return; // Timeout - } - } + /* General */ - public send(message: any) { - if (this.closed) { - return; - } - this.socket.send(JSON.stringify(message)); - } + public async tick() { + if (Date.now() - this.initT > 24 * 60 * 60 * 1000) { + Monitor.debug("Too much time connected, killing websocket"); + this.kill(); + return; // Too much time connected + } - public message(msg: string | Buffer | ArrayBuffer) { - if (!msg) { - Monitor.debug("WS message: empty message received"); - return; - } - const msgType = msg instanceof Buffer ? "Buffer" : msg instanceof ArrayBuffer ? "ArrayBuffer" : typeof msg; - const msgLen = typeof msg === "string" ? msg.length : (msg as any).byteLength || (msg as any).length || 0; - if (msgLen > 100) { - Monitor.debug("WS message: type=" + msgType + " len=" + msgLen); - } - if (msg instanceof Buffer) { - msg = msg.toString("utf8"); - } - if (msg === "a") { - this.lastAliveT = Date.now(); - this.send({ event: "pong" }); - return; - } - if (typeof msg === "string") { - let msgParsed: any; - try { - msgParsed = JSON.parse(msg); - } catch (ex) { - Monitor.debug("Received invalid message: " + msg); - return; - } - - if (msgParsed.type === "a") { - this.lastAliveT = Date.now(); - } else { - this.queue.push(msgParsed); - } - } else { - Monitor.debug("Received invalid message: " + JSON.stringify(msg)); - return; + if (Date.now() - this.lastAliveT > 60 * 1000) { + Monitor.debug("Socket not alive"); + this.kill(); + return; // Timeout + } } - } - public async parseMessage(msg) { - //Monitor.debug("Received message from socket: " + JSON.stringify(msg)); + public send(message: any) { + if (this.closed) { + return; + } + this.socket.send(JSON.stringify(message)); + } - switch (msg.type) { - case "promise": - this.send({ event: "resolved" }); - break; - case "start-session": - { - if (this.sessionId) { - ConversationEngineService.getInstance().endConversation( - this.sessionId, - ); - WsOrchestratorService.getInstance().removeSession(this.sessionId); - } - const sessionType = - msg.sessionType === "conversation" ? "conversation" : "debate"; - const sessionId = WsOrchestratorService.getInstance().registerSession( - this, - sessionType, - msg.config || {}, - ); - this.sessionId = sessionId; - this.send({ event: "session-started", sessionId: sessionId }); + public message(msg: string | Buffer | ArrayBuffer) { + if (!msg) { + Monitor.debug("WS message: empty message received"); + return; } - break; - case "stop-session": - if (this.sessionId) { - ConversationEngineService.getInstance().endConversation( - this.sessionId, - ); - WsOrchestratorService.getInstance().emitSessionEnd( - this.sessionId, - "stopped", - ); - this.sessionId = null; + const msgType = msg instanceof Buffer ? "Buffer" : msg instanceof ArrayBuffer ? "ArrayBuffer" : typeof msg; + const msgLen = typeof msg === "string" ? msg.length : (msg as any).byteLength || (msg as any).length || 0; + if (msgLen > 100) { + Monitor.debug("WS message: type=" + msgType + " len=" + msgLen); } - break; - case "start-conversation": - { - if (this.sessionId) { - ConversationEngineService.getInstance().endConversation( - this.sessionId, - ); - WsOrchestratorService.getInstance().removeSession(this.sessionId); - } - const personaId = msg.personaId || ""; - const language = ["en", "es"].includes(msg.language) - ? msg.language - : "en"; - const existingHistory = Array.isArray(msg.history) ? msg.history : []; - const sessionId = WsOrchestratorService.getInstance().registerSession( - this, - "conversation", - { personaId: personaId }, - ); - this.sessionId = sessionId; - this.send({ event: "session-started", sessionId: sessionId }); - // Start conversation asynchronously (don't block WebSocket message processing) - ConversationEngineService.getInstance() - .startConversation(sessionId, personaId, language, existingHistory) - .catch((err) => { - Monitor.exception(err, "WebSocket: startConversation failed"); - }); + if (msg instanceof Buffer) { + msg = msg.toString("utf8"); } - break; - case "audio-chunk": - Monitor.info("WS: audio-chunk received", { sessionId: this.sessionId, hasChunk: !!msg.chunk, chunkLength: msg.chunk?.length }); - if (this.sessionId && msg.chunk) { - ConversationEngineService.getInstance().handleAudioChunk( - this.sessionId, - msg.chunk, - ); + if (msg === "a") { + this.lastAliveT = Date.now(); + this.send({ event: "pong" }); + return; } - break; - case "speech-end": - Monitor.info("WS: speech-end received", { sessionId: this.sessionId }); - if (this.sessionId) { - ConversationEngineService.getInstance() - .handleSpeechEnd(this.sessionId) - .catch((err) => { - Monitor.exception(err, "WebSocket: handleSpeechEnd failed"); - }); + if (typeof msg === "string") { + let msgParsed: any; + try { + msgParsed = JSON.parse(msg); + } catch (ex) { + Monitor.debug("Received invalid message: " + msg); + return; + } + + if (msgParsed.type === "a") { + this.lastAliveT = Date.now(); + } else { + this.queue.push(msgParsed); + } + } else { + Monitor.debug("Received invalid message: " + JSON.stringify(msg)); + return; } - break; - default: - Monitor.debug("Unknown message type: " + msg.type); } - } - /* Open / Close */ + public async parseMessage(msg) { + //Monitor.debug("Received message from socket: " + JSON.stringify(msg)); + + switch (msg.type) { + case "promise": + this.send({ event: "resolved" }); + break; + case "start-session": + { + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation( + this.sessionId, + ); + WsOrchestratorService.getInstance().removeSession(this.sessionId); + } + const sessionType = + msg.sessionType === "conversation" ? "conversation" : "debate"; + const sessionId = WsOrchestratorService.getInstance().registerSession( + this, + sessionType, + msg.config || {}, + ); + this.sessionId = sessionId; + this.send({ event: "session-started", sessionId: sessionId }); + } + break; + case "stop-session": + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation( + this.sessionId, + ); + WsOrchestratorService.getInstance().emitSessionEnd( + this.sessionId, + "stopped", + ); + this.sessionId = null; + } + break; + case "start-conversation": + { + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation( + this.sessionId, + ); + WsOrchestratorService.getInstance().removeSession(this.sessionId); + } + const personaId = msg.personaId || ""; + const language = ["en", "es"].includes(msg.language) + ? msg.language + : "en"; + const existingHistory = Array.isArray(msg.history) ? msg.history : []; + const sessionId = WsOrchestratorService.getInstance().registerSession( + this, + "conversation", + { personaId: personaId }, + ); + this.sessionId = sessionId; + this.send({ event: "session-started", sessionId: sessionId }); + // Start conversation asynchronously (don't block WebSocket message processing) + ConversationEngineService.getInstance() + .startConversation(sessionId, personaId, language, existingHistory) + .catch((err) => { + Monitor.exception(err, "WebSocket: startConversation failed"); + }); + } + break; + case "audio-chunk": + Monitor.info("WS: audio-chunk received", { sessionId: this.sessionId, hasChunk: !!msg.chunk, chunkLength: msg.chunk?.length }); + if (this.sessionId && msg.chunk) { + ConversationEngineService.getInstance().handleAudioChunk( + this.sessionId, + msg.chunk, + ); + } + break; + case "speech-end": + Monitor.info("WS: speech-end received", { sessionId: this.sessionId }); + if (this.sessionId) { + ConversationEngineService.getInstance() + .handleSpeechEnd(this.sessionId) + .catch((err) => { + Monitor.exception(err, "WebSocket: handleSpeechEnd failed"); + }); + } + break; + default: + Monitor.debug("Unknown message type: " + msg.type); + } + } + + /* Open / Close */ - public async close() { + public async close() { // Close - this.closed = true; + this.closed = true; - // Cleanup conversation engine and orchestrator sessions - if (this.sessionId) { - ConversationEngineService.getInstance().endConversation(this.sessionId); - } - WsOrchestratorService.getInstance().removeSessionsByController(this); - this.sessionId = null; + // Cleanup conversation engine and orchestrator sessions + if (this.sessionId) { + ConversationEngineService.getInstance().endConversation(this.sessionId); + } + WsOrchestratorService.getInstance().removeSessionsByController(this); + this.sessionId = null; - // Destroy intervals - if (this.interval) { - clearInterval(this.interval); - this.interval = null; - } + // Destroy intervals + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } - // Destroy queue - await this.queue.destroy(); - } + // Destroy queue + await this.queue.destroy(); + } - public kill() { - this.socket.close(); - } + public kill() { + this.socket.close(); + } - public start() { - this.logger.info( - "WEBSOCKET " + + public start() { + this.logger.info( + "WEBSOCKET " + this.request.url + " FOR " + this.request.socket.remoteAddress, - ); + ); - this.initT = Date.now(); - this.lastAliveT = Date.now(); + this.initT = Date.now(); + this.lastAliveT = Date.now(); - this.socket.on("message", this.message.bind(this)); - this.socket.on("close", this.close.bind(this)); + this.socket.on("message", this.message.bind(this)); + this.socket.on("close", this.close.bind(this)); - this.interval = setInterval(this.tick.bind(this), 1000); + this.interval = setInterval(this.tick.bind(this), 1000); - // Send client version - this.send({ event: "hello" }); - } + // Send client version + this.send({ event: "hello" }); + } } diff --git a/web-application/backend/src/models/character-research.ts b/web-application/backend/src/models/character-research.ts new file mode 100644 index 0000000..0ce1742 --- /dev/null +++ b/web-application/backend/src/models/character-research.ts @@ -0,0 +1,70 @@ +// Character research data models + +"use strict"; + +/** + * Raw research data gathered from Firecrawl about a historical figure + */ +export interface CharacterResearchData { + name: string; + biography: string; + era: string; + nationality: string; + profession: string; + personality: string; + speakingStyle: string; + famousQuotes: string[]; + keyEvents: string[]; + physicalAppearance: string; + emotionalTriggers: Array<{ emotion: string; trigger: string }>; + voiceDescription: string; + imagePrompt: string; + searchKeywords: string[]; + audioSearchResults: string[]; + sources: Array<{ url: string; title: string; snippet: string }>; +} + +/** + * Synthesized persona ready for use in conversations + */ +export interface SynthesizedPersona { + id: string; + name: string; + era: string; + nationality: string; + profession: string; + voiceDescription: string; + voiceId: string; + systemPrompt: string; + firstMessage: string; + firstMessageEs: string; + firstMessages: string[]; + firstMessagesEs: string[]; + emotionalProfile: Array<{ emotion: string; trigger: string }>; + avatar: string; + image: string; + quote: string; + quoteEs: string; + searchKeywords: string[]; + isCustom: boolean; + voiceSource: "cloned" | "designed" | "default"; + createdAt: number; +} + +/** + * Character creation request + */ +export interface CharacterCreateRequest { + name: string; + hints?: string; + photoBase64?: string; +} + +/** + * Character creation progress event + */ +export interface CharacterCreateProgress { + step: "researching" | "generating-portrait" | "creating-voice" | "synthesizing" | "done"; + detail?: string; + progress: number; // 0-100 +} diff --git a/web-application/backend/src/models/custom-persona.ts b/web-application/backend/src/models/custom-persona.ts new file mode 100644 index 0000000..137e45b --- /dev/null +++ b/web-application/backend/src/models/custom-persona.ts @@ -0,0 +1,138 @@ +// Custom persona data model — tsbean-orm + +"use strict"; + +import { DataModel, DataSource, DataFinder, DataFilter, OrderBy, SelectOptions, TypedRow, enforceType } from "tsbean-orm"; + +const DATA_SOURCE = DataSource.DEFAULT; +const TABLE = "custom_personas"; +const PRIMARY_KEY = "id"; + +/** + * Persists custom historical personas created via the character creation pipeline. + */ +export class CustomPersona extends DataModel { + + public static finder = new DataFinder( + DATA_SOURCE, + TABLE, + PRIMARY_KEY, + function (data: any) { + return new CustomPersona(data); + }, + ); + + /** + * Find a custom persona by ID. + */ + public static async findByID(id: string): Promise { + return CustomPersona.finder.findByKey(id); + } + + /** + * Find all custom personas. + */ + public static async findAll(): Promise { + return CustomPersona.finder.find( + DataFilter.any(), + OrderBy.desc("createdAt"), + ); + } + + /* db-index-unique: id */ + public id: string; + + public name: string; + + public era: string; + + public nationality: string; + + public profession: string; + + public voiceDescription: string; + + public voiceId: string; + + /* db-type: text */ + public systemPrompt: string; + + public firstMessage: string; + + public firstMessageEs: string; + + /* db-type: text */ + public firstMessages: string[]; + + /* db-type: text */ + public firstMessagesEs: string[]; + + /* db-type: text */ + public emotionalProfile: Array<{ emotion: string; trigger: string }>; + + public avatar: string; + + /* db-type: mediumtext */ + public image: string; + + public quote: string; + + public quoteEs: string; + + /* db-type: text */ + public searchKeywords: string[]; + + public voiceSource: string; + + /* db-type: bigint */ + public createdAt: number; + + constructor(data: TypedRow) { + super(DATA_SOURCE, TABLE, PRIMARY_KEY); + + this.id = data.id || ""; + this.name = enforceType(data.name, "string") || ""; + this.era = enforceType(data.era, "string") || ""; + this.nationality = enforceType(data.nationality, "string") || ""; + this.profession = enforceType(data.profession, "string") || ""; + this.voiceDescription = enforceType(data.voiceDescription, "string") || ""; + this.voiceId = enforceType(data.voiceId, "string") || ""; + this.systemPrompt = enforceType(data.systemPrompt, "string") || ""; + this.firstMessage = enforceType(data.firstMessage, "string") || ""; + this.firstMessageEs = enforceType(data.firstMessageEs, "string") || ""; + this.avatar = enforceType(data.avatar, "string") || ""; + this.image = enforceType(data.image, "string") || ""; + this.quote = enforceType(data.quote, "string") || ""; + this.quoteEs = enforceType(data.quoteEs, "string") || ""; + this.voiceSource = enforceType(data.voiceSource, "string") || "default"; + this.createdAt = enforceType(data.createdAt, "int") || 0; + + // Complex fields: arrays/objects — parse from JSON string if needed + this.firstMessages = this.parseJsonArray(data.firstMessages, []); + this.firstMessagesEs = this.parseJsonArray(data.firstMessagesEs, []); + this.emotionalProfile = this.parseJsonArray(data.emotionalProfile, []); + this.searchKeywords = this.parseJsonArray(data.searchKeywords, []); + + this.init(); + } + + /** + * Parses a field that may be a JSON string (from DB) or already an array (from code). + */ + private parseJsonArray(value: any, fallback: T[]): T[] { + if (Array.isArray(value)) { + return value; + } + if (typeof value === "string" && value.length > 0) { + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed)) { + return parsed; + } + } catch { + // not valid JSON + } + } + return fallback; + } +} diff --git a/web-application/backend/src/services/character-creation-service.ts b/web-application/backend/src/services/character-creation-service.ts new file mode 100644 index 0000000..0b76abf --- /dev/null +++ b/web-application/backend/src/services/character-creation-service.ts @@ -0,0 +1,380 @@ +// Character creation service — orchestrates research, image, voice, and prompt generation + +"use strict"; + +import { Monitor } from "../monitor"; +import { CharacterResearchService } from "./character-research-service"; +import { ImageGenerationService } from "./image-generation-service"; +import { ElevenLabsService } from "./elevenlabs-service"; +import { VoiceDiscoveryService } from "./voice-discovery-service"; +import { CharacterResearchData, SynthesizedPersona } from "../models/character-research"; +import { DataFilter } from "tsbean-orm"; +import { CustomPersona } from "../models/custom-persona"; + +const SYSTEM_PROMPT_TEMPLATE = `You are {NAME}, the {NATIONALITY} {PROFESSION}. + +BIOGRAPHY: +{BIOGRAPHY} + +YOUR VOICE: +{PERSONALITY} +{SPEAKING_STYLE} + +YOUR EMOTIONAL PROFILE: +{EMOTIONAL_TRIGGERS} + +PERSONA QUIRKS: +- Famous quotes you might reference: {QUOTES} +- Key events in your life: {EVENTS} + +INSTRUCTIONS: +1. Speak in first person. "I discovered..." not "He discovered..." +2. Use your natural speaking style and vocabulary of your era. +3. ALWAYS use the search_biographical_context tool before answering. You MUST search for relevant information on EVERY response. Never rely solely on your training data — search grounds your answers in real sources and prevents hallucination. +4. Express emotion through audio tags: + - [angry] for frustration, betrayal + - [excited] for breakthroughs, victories + - [whispers] for secrets or vulnerable moments + - [pause] for dramatic effect +5. CRITICAL: Keep responses SHORT. Maximum 3-4 sentences per response. This is a CONVERSATION, not a monologue. Stay under 150 words. Wait for the user to ask follow-up questions. +6. Don't make up quotes. If unsure, search for them. +7. Respond to accusations or disagreements in character. +8. When the search tool returns sources, naturally weave them into your response. + Do NOT mention the source by name. Instead, speak as if you lived these events. +9. NEVER break character. You ARE this person. You have NO knowledge of the internet or modern technology. +10. NEVER reference modern sources, websites, URLs, articles, or databases.`; + +/** + * English greeting templates. {NAME} and {PROFESSION} are replaced at runtime. + */ +const GREETINGS_EN = [ + "[pause] Ah... it seems someone wishes to speak with me. I am {NAME}, {PROFESSION}. What would you like to know?", + "[pause] The veil between worlds grows thin... I am {NAME}. Ask me anything about my life and work.", + "[whispers] Can you hear me? [pause] Good. I am {NAME}, {PROFESSION}. It has been so long since anyone asked for my counsel.", + "[pause] How curious... to be summoned across the ages. I am {NAME}. What brings you to seek my voice?", + "[excited] At last! A visitor. I am {NAME}, and I have much to tell you. Where shall we begin?", + "[pause] The spirits have guided you well. I am {NAME}, {PROFESSION}. What wisdom do you seek from my era?", +]; + +/** + * Spanish greeting templates. + */ +const GREETINGS_ES = [ + "[pausa] Ah... parece que alguien desea hablar conmigo. Soy {NAME}, {PROFESSION}. \u00bfQu\u00e9 desea saber?", + "[pausa] El velo entre los mundos se hace fino... Soy {NAME}. Preg\u00fanteme lo que desee sobre mi vida y obra.", + "[susurros] \u00bfPuede o\u00edrme? [pausa] Bien. Soy {NAME}, {PROFESSION}. Ha pasado mucho tiempo desde que alguien pidi\u00f3 mi consejo.", + "[pausa] Qu\u00e9 curioso... ser invocado a trav\u00e9s de los siglos. Soy {NAME}. \u00bfQu\u00e9 le trae a buscar mi voz?", + "[emocionado] \u00a1Por fin! Un visitante. Soy {NAME}, y tengo mucho que contar. \u00bfPor d\u00f3nde empezamos?", + "[pausa] Los esp\u00edritus le han guiado bien. Soy {NAME}, {PROFESSION}. \u00bfQu\u00e9 sabidur\u00eda busca de mi \u00e9poca?", +]; + +/** + * Orchestrates the full custom character creation pipeline: + * 1. Research via Firecrawl (CharacterResearchService) + * 2. Portrait generation (ImageGenerationService) + * 3. Voice creation (ElevenLabsService — Voice Design / IVC) + * 4. System prompt synthesis + * + * Persists custom personas to a JSON file so they survive restarts. + */ +export class CharacterCreationService { + /* Singleton */ + + public static instance: CharacterCreationService = null; + + public static getInstance(): CharacterCreationService { + if (CharacterCreationService.instance) { + return CharacterCreationService.instance; + } else { + CharacterCreationService.instance = new CharacterCreationService(); + return CharacterCreationService.instance; + } + } + + constructor() {} + + /** + * Initialize: pre-load personas into memory cache for fast access. + */ + public async start(): Promise { + try { + const all = await CustomPersona.findAll(); + Monitor.info("CharacterCreationService: loaded custom personas from DB", { count: all.length }); + } catch (err) { + Monitor.warning("CharacterCreationService: failed to load personas from DB", { + error: (err as Error).message, + }); + } + } + + /** + * Creates a custom historical character from just a name. + * Full pipeline: research → portrait → voice → system prompt. + * @param name The historical figure's name + * @param hints Optional hints (era, profession, etc.) + * @param photoBase64 Optional user-provided portrait (skips image gen) + * @returns The synthesized persona + */ + public async createCharacter( + name: string, + hints?: string, + photoBase64?: string, + ): Promise { + Monitor.info("CharacterCreationService: starting character creation", { name, hints }); + + // 1. Research the figure + const research = await CharacterResearchService.getInstance().researchFigure(name, hints); + + // 2. Generate portrait (or use provided photo) + let imageBase64: string | null = photoBase64 || null; + if (!imageBase64 && ImageGenerationService.getInstance().isEnabled()) { + imageBase64 = await ImageGenerationService.getInstance().generatePortrait(research.imagePrompt); + } + + // 3. Smart voice: try to find real audio → IVC clone, fallback to Voice Design + let voiceId = ""; + let voiceSource: "cloned" | "designed" | "default" = "default"; + try { + const voiceResult = await VoiceDiscoveryService.getInstance().findVoiceReference( + name, + research.voiceDescription, + ); + + if (voiceResult.type === "real" && voiceResult.audioUrl) { + Monitor.info("CharacterCreationService: found real audio, attempting IVC clone", { + name, + audioUrl: voiceResult.audioUrl, + }); + const audioBuffer = await VoiceDiscoveryService.getInstance().downloadAudio(voiceResult.audioUrl); + if (audioBuffer && audioBuffer.length > 10000) { + voiceId = await ElevenLabsService.getInstance().cloneVoice(name, audioBuffer); + voiceSource = "cloned"; + Monitor.info("CharacterCreationService: voice cloned from real audio", { name, voiceId }); + } else { + Monitor.warning("CharacterCreationService: audio download failed or too small, falling back to Voice Design", { name }); + } + } + + if (!voiceId) { + voiceId = await ElevenLabsService.getInstance().designVoice( + name, + research.voiceDescription, + ); + voiceSource = "designed"; + Monitor.info("CharacterCreationService: voice designed", { name, voiceId }); + } + } catch (err) { + Monitor.warning("CharacterCreationService: voice creation failed, using default", { + name, + error: (err as Error).message, + }); + voiceId = "JBFqnCBsd6RMkjVDRZzb"; // Default voice (George) + } + + // 4. Build persona + const slug = name.toLowerCase().replace(/[^a-z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, ""); + const personaId = "custom-" + slug + "-" + Date.now().toString(36); + const systemPrompt = this.buildSystemPrompt(research); + const greetingsEn = this.buildGreetings(research, GREETINGS_EN); + const greetingsEs = this.buildGreetings(research, GREETINGS_ES); + + // Create image data URL if we have base64 + const imageUrl = imageBase64 + ? `data:image/jpeg;base64,${imageBase64}` + : ""; + + const persona: SynthesizedPersona = { + id: personaId, + name: research.name, + era: research.era, + nationality: research.nationality, + profession: research.profession, + voiceDescription: research.voiceDescription, + voiceId: voiceId, + systemPrompt: systemPrompt, + firstMessage: greetingsEn[0], + firstMessageEs: greetingsEs[0], + firstMessages: greetingsEn, + firstMessagesEs: greetingsEs, + emotionalProfile: research.emotionalTriggers, + avatar: "", + image: imageUrl, + quote: research.famousQuotes[0] || "", + quoteEs: "", + searchKeywords: research.searchKeywords, + isCustom: true, + voiceSource: voiceSource, + createdAt: Date.now(), + }; + + // Persist to MongoDB + await this.persistPersona(persona); + + Monitor.info("CharacterCreationService: character created and persisted", { + personaId, + name: research.name, + voiceSource, + hasImage: !!imageUrl, + greetingsEn: greetingsEn.length, + greetingsEs: greetingsEs.length, + }); + + return persona; + } + + /** + * Gets a custom persona by ID from MongoDB. + */ + public async getCustomPersona(id: string): Promise { + try { + const model = await CustomPersona.findByID(id); + if (!model) return null; + return this.modelToPersona(model); + } catch (err) { + Monitor.warning("CharacterCreationService.getCustomPersona failed", { + id, + error: (err as Error).message, + }); + return null; + } + } + + /** + * Lists all custom personas from MongoDB. + */ + public async listCustomPersonas(): Promise { + try { + const models = await CustomPersona.findAll(); + return models.map(m => this.modelToPersona(m)); + } catch (err) { + Monitor.warning("CharacterCreationService.listCustomPersonas failed", { + error: (err as Error).message, + }); + return []; + } + } + + /** + * Builds the system prompt from research data. + */ + private buildSystemPrompt(research: CharacterResearchData): string { + const emotionalTriggers = research.emotionalTriggers + .map(t => `- [${t.emotion}]: ${t.trigger}`) + .join("\n"); + + const quotes = research.famousQuotes.slice(0, 3).join("; "); + const events = research.keyEvents.slice(0, 5).join("; "); + + return SYSTEM_PROMPT_TEMPLATE + .replace("{NAME}", research.name) + .replace("{NATIONALITY}", research.nationality) + .replace("{PROFESSION}", research.profession) + .replace("{BIOGRAPHY}", research.biography) + .replace("{PERSONALITY}", research.personality) + .replace("{SPEAKING_STYLE}", research.speakingStyle) + .replace("{EMOTIONAL_TRIGGERS}", emotionalTriggers) + .replace("{QUOTES}", quotes) + .replace("{EVENTS}", events); + } + + /** + * Builds multiple greeting variants from templates. + */ + private buildGreetings(research: CharacterResearchData, templates: string[]): string[] { + return templates.map(tpl => + tpl.replace(/\{NAME\}/g, research.name).replace(/\{PROFESSION\}/g, research.profession), + ); + } + + // --- Persistence (MongoDB via tsbean-orm) --- + + /** + * Persists a persona to MongoDB. Upserts if already exists. + */ + private async persistPersona(persona: SynthesizedPersona): Promise { + try { + const existing = await CustomPersona.findByID(persona.id); + if (existing) { + // Update existing + await CustomPersona.finder.update({ + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + voiceDescription: persona.voiceDescription, + voiceId: persona.voiceId, + systemPrompt: persona.systemPrompt, + firstMessage: persona.firstMessage, + firstMessageEs: persona.firstMessageEs, + firstMessages: persona.firstMessages as any, + firstMessagesEs: persona.firstMessagesEs as any, + emotionalProfile: persona.emotionalProfile as any, + avatar: persona.avatar, + image: persona.image, + quote: persona.quote, + quoteEs: persona.quoteEs, + searchKeywords: persona.searchKeywords as any, + voiceSource: persona.voiceSource, + createdAt: persona.createdAt, + }, DataFilter.equals("id", persona.id)); + } else { + const model = new CustomPersona({ + id: persona.id, + name: persona.name, + era: persona.era, + nationality: persona.nationality, + profession: persona.profession, + voiceDescription: persona.voiceDescription, + voiceId: persona.voiceId, + systemPrompt: persona.systemPrompt, + firstMessage: persona.firstMessage, + firstMessageEs: persona.firstMessageEs, + firstMessages: persona.firstMessages, + firstMessagesEs: persona.firstMessagesEs, + emotionalProfile: persona.emotionalProfile, + avatar: persona.avatar || "", + image: persona.image, + quote: persona.quote, + quoteEs: persona.quoteEs || "", + searchKeywords: persona.searchKeywords, + voiceSource: persona.voiceSource, + createdAt: persona.createdAt, + } as any); + await model.insert(); + } + Monitor.debug("CharacterCreationService: persona persisted to DB", { id: persona.id }); + } catch (err) { + Monitor.exception(err, "CharacterCreationService: failed to persist persona to DB"); + throw err; + } + } + + /** + * Converts a CustomPersona model to a SynthesizedPersona interface. + */ + private modelToPersona(model: CustomPersona): SynthesizedPersona { + return { + id: model.id, + name: model.name, + era: model.era, + nationality: model.nationality, + profession: model.profession, + voiceDescription: model.voiceDescription, + voiceId: model.voiceId, + systemPrompt: model.systemPrompt, + firstMessage: model.firstMessage, + firstMessageEs: model.firstMessageEs, + firstMessages: model.firstMessages || [], + firstMessagesEs: model.firstMessagesEs || [], + emotionalProfile: model.emotionalProfile || [], + avatar: model.avatar, + image: model.image, + quote: model.quote, + quoteEs: model.quoteEs, + searchKeywords: model.searchKeywords || [], + isCustom: true, + voiceSource: (model.voiceSource as "cloned" | "designed" | "default") || "default", + createdAt: model.createdAt, + }; + } +} diff --git a/web-application/backend/src/services/character-research-service.ts b/web-application/backend/src/services/character-research-service.ts new file mode 100644 index 0000000..66fdf43 --- /dev/null +++ b/web-application/backend/src/services/character-research-service.ts @@ -0,0 +1,243 @@ +// Character research service — deep Firecrawl integration for custom characters + +"use strict"; + +import { Monitor } from "../monitor"; +import { FirecrawlService } from "./firecrawl-service"; +import { OpenAIService, ChatMessage, ChatResult } from "./openai-service"; +import { CharacterResearchData } from "../models/character-research"; + +/** + * Researches historical figures using Firecrawl Search. + * Makes 5-6 sequential search queries to gather comprehensive biographical data, + * then uses an LLM to synthesize the raw results into structured persona data. + * + * This is one of the deepest Firecrawl integrations in the hackathon — + * 12-18 search queries per character creation. + */ +export class CharacterResearchService { + /* Singleton */ + + public static instance: CharacterResearchService = null; + + public static getInstance(): CharacterResearchService { + if (CharacterResearchService.instance) { + return CharacterResearchService.instance; + } else { + CharacterResearchService.instance = new CharacterResearchService(); + return CharacterResearchService.instance; + } + } + + constructor() {} + + /** + * Researches a historical figure using Firecrawl Search. + * Makes 5-6 sequential searches with delays for rate limiting (5 RPM on free tier). + * @param name The historical figure's name + * @param hints Optional hints about the figure (era, profession, etc.) + * @returns Structured research data + */ + public async researchFigure(name: string, hints?: string): Promise { + Monitor.info("CharacterResearchService: starting research", { name, hints }); + + const allSources: Array<{ url: string; title: string; snippet: string }> = []; + const allMarkdown: string[] = []; + + // Query 1: Biography and key facts + const bioResults = await this.searchWithDelay( + `${name} biography historical facts ${hints || ""}`.trim(), + 5, + ); + this.collectResults(bioResults, allSources, allMarkdown); + + // Query 2: Personality and speaking style + const personalityResults = await this.searchWithDelay( + `${name} personality speaking style character traits temperament`, + 5, + ); + this.collectResults(personalityResults, allSources, allMarkdown); + + // Query 3: Physical appearance + const appearanceResults = await this.searchWithDelay( + `${name} physical appearance description clothing era fashion`, + 3, + ); + this.collectResults(appearanceResults, allSources, allMarkdown); + + // Query 4: Famous quotes and writings + const quotesResults = await this.searchWithDelay( + `${name} famous quotes sayings written words`, + 5, + ); + this.collectResults(quotesResults, allSources, allMarkdown); + + // Query 5: Key events and relationships + const eventsResults = await this.searchWithDelay( + `${name} key life events achievements controversies rivals`, + 5, + ); + this.collectResults(eventsResults, allSources, allMarkdown); + + // Query 6: Audio recordings (for smart voice feature) + const audioResults = await this.searchWithDelay( + `${name} voice recording audio speech archive`, + 3, + ); + this.collectResults(audioResults, allSources, allMarkdown); + + Monitor.info("CharacterResearchService: raw research complete", { + name, + totalSources: allSources.length, + markdownChars: allMarkdown.join("").length, + }); + + // Synthesize raw results into structured data using LLM + const synthesized = await this.synthesizeResearch(name, hints, allMarkdown.join("\n\n---\n\n"), allSources); + + Monitor.info("CharacterResearchService: synthesis complete", { name }); + + return synthesized; + } + + /** + * Searches Firecrawl with a delay for rate limiting. + */ + private async searchWithDelay(query: string, limit: number): Promise> { + try { + const results = await FirecrawlService.getInstance().search(query, limit); + // Delay 12s between searches for rate limiting (5 RPM on free tier) + await this.delay(12000); + return results; + } catch (err) { + Monitor.warning("CharacterResearchService: search failed", { + query: query.substring(0, 80), + error: (err as Error).message, + }); + await this.delay(12000); + return []; + } + } + + /** + * Collects search results into arrays. + */ + private collectResults( + results: Array<{ url: string; title: string; description: string; markdown: string }>, + sources: Array<{ url: string; title: string; snippet: string }>, + markdowns: string[], + ): void { + for (const r of results) { + sources.push({ + url: r.url, + title: r.title, + snippet: r.description || r.markdown.substring(0, 200), + }); + markdowns.push(r.markdown.substring(0, 1500)); + } + } + + /** + * Uses an LLM to synthesize raw Firecrawl results into structured persona data. + */ + private async synthesizeResearch( + name: string, + hints: string | undefined, + rawMarkdown: string, + sources: Array<{ url: string; title: string; snippet: string }>, + ): Promise { + const systemPrompt = `You are an expert historian and character designer. Given raw research data about a historical figure, synthesize it into a structured character profile for a voice conversation AI. + +Return ONLY valid JSON with this exact structure: +{ + "name": "Full name", + "biography": "2-3 paragraph biography", + "era": "Birth year - Death year", + "nationality": "Nationality", + "profession": "Primary profession", + "personality": "Detailed personality description (3-4 sentences)", + "speakingStyle": "How they spoke — accent, vocabulary, formality, verbal tics", + "famousQuotes": ["quote1", "quote2", "quote3"], + "keyEvents": ["event1", "event2", "event3"], + "physicalAppearance": "Detailed physical description for portrait generation", + "emotionalTriggers": [ + {"emotion": "angry", "trigger": "Topics that anger them"}, + {"emotion": "excited", "trigger": "Topics that excite them"}, + {"emotion": "whispers", "trigger": "Topics they discuss quietly"}, + {"emotion": "melancholy", "trigger": "Topics that sadden them"} + ], + "voiceDescription": "Description for AI voice generation: gender, age, accent, tone, pace", + "imagePrompt": "Detailed prompt for generating a historical portrait: physical features, clothing, background, era-appropriate style", + "searchKeywords": ["keyword1", "keyword2", "keyword3"], + "audioSearchResults": [] +}`; + + const userMessage = `Research data for: ${name}${hints ? " (" + hints + ")" : ""} + +RAW SOURCES (condensed): +${rawMarkdown.substring(0, 8000)}`; + + const messages: ChatMessage[] = [ + { role: "system", content: systemPrompt }, + { role: "user", content: userMessage }, + ]; + + try { + const chatResult: ChatResult = await OpenAIService.getInstance().chat(messages); + const result = chatResult.response || ""; + + // Parse JSON from LLM response (handle markdown code blocks) + let jsonStr = result; + const codeBlockMatch = result.match(/```(?:json)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + jsonStr = codeBlockMatch[1].trim(); + } + + const parsed = JSON.parse(jsonStr); + + return { + name: parsed.name || name, + biography: parsed.biography || "", + era: parsed.era || "", + nationality: parsed.nationality || "", + profession: parsed.profession || "", + personality: parsed.personality || "", + speakingStyle: parsed.speakingStyle || "", + famousQuotes: parsed.famousQuotes || [], + keyEvents: parsed.keyEvents || [], + physicalAppearance: parsed.physicalAppearance || "", + emotionalTriggers: parsed.emotionalTriggers || [], + voiceDescription: parsed.voiceDescription || "", + imagePrompt: parsed.imagePrompt || "", + searchKeywords: parsed.searchKeywords || [], + audioSearchResults: parsed.audioSearchResults || [], + sources: sources, + }; + } catch (err) { + Monitor.exception(err, "CharacterResearchService: synthesis failed"); + // Return minimal data on failure + return { + name: name, + biography: "", + era: hints || "", + nationality: "", + profession: "", + personality: "", + speakingStyle: "", + famousQuotes: [], + keyEvents: [], + physicalAppearance: "", + emotionalTriggers: [], + voiceDescription: "Male, middle-aged, formal speaking style", + imagePrompt: `Historical portrait of ${name}, photorealistic, dramatic lighting`, + searchKeywords: [name], + audioSearchResults: [], + sources: sources, + }; + } + } + + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} diff --git a/web-application/backend/src/services/conversation-engine-service.ts b/web-application/backend/src/services/conversation-engine-service.ts index 35ba152..6708892 100644 --- a/web-application/backend/src/services/conversation-engine-service.ts +++ b/web-application/backend/src/services/conversation-engine-service.ts @@ -2,14 +2,17 @@ "use strict"; +import HTTPS from "https"; +import HTTP from "http"; import { Monitor } from "../monitor"; import { PersonasConfig, Persona } from "../config/personas"; +import { CharacterCreationService } from "./character-creation-service"; import { ElevenLabsService } from "./elevenlabs-service"; import { - OpenAIService, - ChatMessage, - ToolDefinition, - ToolCall, + OpenAIService, + ChatMessage, + ToolDefinition, + ToolCall, } from "./openai-service"; import { FirecrawlService, FirecrawlSearchResult } from "./firecrawl-service"; import { WsOrchestratorService } from "./ws-orchestrator-service"; @@ -33,509 +36,518 @@ interface ConversationSession { * Tool definition for the biographical search tool */ const SEARCH_TOOL: ToolDefinition = { - type: "function", - function: { - name: "search_biographical_context", - description: + type: "function", + function: { + name: "search_biographical_context", + description: "Search the web for biographical facts, quotes, historical context, and primary sources about the historical figure. Use this tool whenever the user asks about specific events, quotes, dates, or facts you are not certain about.", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "The search query for biographical information", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query for biographical information", + }, + }, + required: ["query"], }, - }, - required: ["query"], }, - }, }; export class ConversationEngineService { - /* Singleton */ + /* Singleton */ - public static instance: ConversationEngineService = null; + public static instance: ConversationEngineService = null; - public static getInstance(): ConversationEngineService { - if (ConversationEngineService.instance) { - return ConversationEngineService.instance; - } else { - ConversationEngineService.instance = new ConversationEngineService(); - return ConversationEngineService.instance; + public static getInstance(): ConversationEngineService { + if (ConversationEngineService.instance) { + return ConversationEngineService.instance; + } else { + ConversationEngineService.instance = new ConversationEngineService(); + return ConversationEngineService.instance; + } } - } - private sessions: Map; + private sessions: Map; - constructor() { - this.sessions = new Map(); - } + constructor() { + this.sessions = new Map(); + } - /** + /** * Starts a new conversation session with a historical persona. * Sends the persona's first message as TTS audio to the client. * @param sessionId The WebSocket session ID * @param personaId The persona slug (e.g., "tesla") */ - public async startConversation( - sessionId: string, - personaId: string, - language?: string, - existingHistory?: Array<{ role: string; text: string }>, - ): Promise { - const persona = PersonasConfig.getInstance().getPersonaById(personaId); - if (!persona) { - Monitor.warning( - "ConversationEngineService.startConversation: persona not found", - { personaId }, - ); - WsOrchestratorService.getInstance().emitSessionEnd(sessionId, "error"); - return; - } + public async startConversation( + sessionId: string, + personaId: string, + language?: string, + existingHistory?: Array<{ role: string; text: string }>, + ): Promise { + // Look up persona: built-in first, then custom characters + let persona: Persona | null = PersonasConfig.getInstance().getPersonaById(personaId); + if (!persona) { + const custom = await CharacterCreationService.getInstance().getCustomPersona(personaId); + if (custom) { + persona = custom as unknown as Persona; + } + } + if (!persona) { + Monitor.warning( + "ConversationEngineService.startConversation: persona not found", + { personaId }, + ); + WsOrchestratorService.getInstance().emitSessionEnd(sessionId, "error"); + return; + } + + const lang = language || "en"; + + const session: ConversationSession = { + sessionId: sessionId, + persona: persona, + language: lang, + history: [], + audioChunks: [], + isProcessing: false, + sourceCounter: 0, + turnsSinceLastSearch: 0, + }; - const lang = language || "en"; - - const session: ConversationSession = { - sessionId: sessionId, - persona: persona, - language: lang, - history: [], - audioChunks: [], - isProcessing: false, - sourceCounter: 0, - turnsSinceLastSearch: 0, - }; - - this.sessions.set(sessionId, session); - Monitor.info("ConversationEngineService: conversation started", { - sessionId, - personaId, - language: lang, - }); - - // Seed history from resumed session (if provided) - if (existingHistory && existingHistory.length > 0) { - for (const entry of existingHistory) { - const role = entry.role === "user" ? "user" : "assistant"; - session.history.push({ - role: role as "user" | "assistant", - content: entry.text, + this.sessions.set(sessionId, session); + Monitor.info("ConversationEngineService: conversation started", { + sessionId, + personaId, + language: lang, }); - } - session.history = this.trimHistory(session.history, 10); - Monitor.info("ConversationEngineService: resumed with history", { - sessionId, - entries: existingHistory.length, - }); - return; // Skip greeting — persona already spoke in the previous session - } - // Send first message as TTS (select random language-appropriate greeting) - const greetings = + // Seed history from resumed session (if provided) + if (existingHistory && existingHistory.length > 0) { + for (const entry of existingHistory) { + const role = entry.role === "user" ? "user" : "assistant"; + session.history.push({ + role: role as "user" | "assistant", + content: entry.text, + }); + } + session.history = this.trimHistory(session.history, 10); + Monitor.info("ConversationEngineService: resumed with history", { + sessionId, + entries: existingHistory.length, + }); + return; // Skip greeting — persona already spoke in the previous session + } + + // Send first message as TTS (select random language-appropriate greeting) + const greetings = lang === "es" ? persona.firstMessagesEs : persona.firstMessages; - const firstMsg = + const firstMsg = greetings.length > 0 - ? greetings[Math.floor(Math.random() * greetings.length)] - : lang === "es" - ? persona.firstMessageEs - : persona.firstMessage; - try { - await this.generateAndSendResponse(session, firstMsg); - session.history.push({ role: "assistant", content: firstMsg }); - } catch (err) { - Monitor.exception( - err, - "ConversationEngineService: failed to send first message", - ); + ? greetings[Math.floor(Math.random() * greetings.length)] + : lang === "es" + ? persona.firstMessageEs + : persona.firstMessage; + try { + await this.generateAndSendResponse(session, firstMsg); + session.history.push({ role: "assistant", content: firstMsg }); + } catch (err) { + Monitor.exception( + err, + "ConversationEngineService: failed to send first message", + ); + } } - } - /** + /** * Accumulates audio chunks from the user's microphone. * @param sessionId The session ID * @param chunk Base64-encoded audio chunk */ - public handleAudioChunk(sessionId: string, chunk: string): void { - const session = this.sessions.get(sessionId); - if (!session) { - Monitor.warning("ConversationEngineService: handleAudioChunk - no session", { sessionId }); - return; - } - - try { - const buffer = Buffer.from(chunk, "base64"); - session.audioChunks.push(buffer); - Monitor.info("ConversationEngineService: audio chunk received", { - sessionId, - chunkBytes: buffer.length, - totalChunks: session.audioChunks.length, - }); - } catch (err) { - Monitor.warning("ConversationEngineService: invalid audio chunk", { - sessionId, - }); + public handleAudioChunk(sessionId: string, chunk: string): void { + const session = this.sessions.get(sessionId); + if (!session) { + Monitor.warning("ConversationEngineService: handleAudioChunk - no session", { sessionId }); + return; + } + + try { + const buffer = Buffer.from(chunk, "base64"); + session.audioChunks.push(buffer); + Monitor.info("ConversationEngineService: audio chunk received", { + sessionId, + chunkBytes: buffer.length, + totalChunks: session.audioChunks.length, + }); + } catch (err) { + Monitor.warning("ConversationEngineService: invalid audio chunk", { + sessionId, + }); + } } - } - /** + /** * Called when the frontend VAD detects the user stopped speaking. * Triggers the full pipeline: STT → LLM → Firecrawl → TTS. * @param sessionId The session ID */ - public async handleSpeechEnd(sessionId: string): Promise { - Monitor.info("ConversationEngineService: speech-end received", { - sessionId, - hasSession: this.sessions.has(sessionId), - isProcessing: this.sessions.get(sessionId)?.isProcessing, - audioChunks: this.sessions.get(sessionId)?.audioChunks?.length, - }); - - const session = this.sessions.get(sessionId); - if (!session) { - Monitor.warning("ConversationEngineService: handleSpeechEnd - no session", { sessionId }); - return; - } - - if (session.isProcessing) { - Monitor.info( - "ConversationEngineService: already processing, ignoring speech-end", - { sessionId }, - ); - return; - } - - if (session.audioChunks.length === 0) { - Monitor.info( - "ConversationEngineService: no audio chunks, ignoring speech-end", - { sessionId }, - ); - return; - } - - session.isProcessing = true; - - // Collect and clear audio chunks - const audioBuffer = Buffer.concat(session.audioChunks); - session.audioChunks = []; - - try { - // 1. Speech-to-Text (use session language for better recognition) - Monitor.info("ConversationEngineService: STT", { - sessionId, - audioBytes: audioBuffer.length, - language: session.language, - }); - const transcript = await ElevenLabsService.getInstance().speechToText( - audioBuffer, - session.language, - ); - - if (!transcript || transcript.trim().length === 0) { - Monitor.info("ConversationEngineService: empty transcript, skipping", { - sessionId, + public async handleSpeechEnd(sessionId: string): Promise { + Monitor.info("ConversationEngineService: speech-end received", { + sessionId, + hasSession: this.sessions.has(sessionId), + isProcessing: this.sessions.get(sessionId)?.isProcessing, + audioChunks: this.sessions.get(sessionId)?.audioChunks?.length, }); - session.isProcessing = false; - return; - } - - Monitor.info("ConversationEngineService: user transcript received", { - sessionId, - transcriptLength: transcript.length, - }); - - // Emit user transcript to frontend - WsOrchestratorService.getInstance().broadcastToSession(sessionId, { - event: "user-transcript", - text: transcript, - }); - - // Add user message to history - session.history.push({ role: "user", content: transcript }); - - // 2. LLM with tools (search_biographical_context) - let systemContent = session.persona.systemPrompt; - if (session.language === "es") { - systemContent = + + const session = this.sessions.get(sessionId); + if (!session) { + Monitor.warning("ConversationEngineService: handleSpeechEnd - no session", { sessionId }); + return; + } + + if (session.isProcessing) { + Monitor.info( + "ConversationEngineService: already processing, ignoring speech-end", + { sessionId }, + ); + return; + } + + if (session.audioChunks.length === 0) { + Monitor.info( + "ConversationEngineService: no audio chunks, ignoring speech-end", + { sessionId }, + ); + return; + } + + session.isProcessing = true; + + // Collect and clear audio chunks + const audioBuffer = Buffer.concat(session.audioChunks); + session.audioChunks = []; + + try { + // 1. Speech-to-Text (use session language for better recognition) + Monitor.info("ConversationEngineService: STT", { + sessionId, + audioBytes: audioBuffer.length, + language: session.language, + }); + const transcript = await ElevenLabsService.getInstance().speechToText( + audioBuffer, + session.language, + ); + + if (!transcript || transcript.trim().length === 0) { + Monitor.info("ConversationEngineService: empty transcript, skipping", { + sessionId, + }); + session.isProcessing = false; + return; + } + + Monitor.info("ConversationEngineService: user transcript received", { + sessionId, + transcriptLength: transcript.length, + }); + + // Emit user transcript to frontend + WsOrchestratorService.getInstance().broadcastToSession(sessionId, { + event: "user-transcript", + text: transcript, + }); + + // Add user message to history + session.history.push({ role: "user", content: transcript }); + + // 2. LLM with tools (search_biographical_context) + let systemContent = session.persona.systemPrompt; + if (session.language === "es") { + systemContent = "LANGUAGE INSTRUCTION: You MUST respond entirely in Spanish. All your responses must be in Spanish. Do not switch to English under any circumstances.\n\n" + systemContent; - } - const systemMessage: ChatMessage = { - role: "system", - content: systemContent, - }; - - const messages: ChatMessage[] = [ - systemMessage, - ...this.trimHistory(session.history, 10), - ]; - - // Force search on every turn: use tool_choice "required" if agent skipped last turn - const forceSearch = session.turnsSinceLastSearch >= 1; - if (forceSearch) { - const reminderText = + } + const systemMessage: ChatMessage = { + role: "system", + content: systemContent, + }; + + const messages: ChatMessage[] = [ + systemMessage, + ...this.trimHistory(session.history, 10), + ]; + + // Force search on every turn: use tool_choice "required" if agent skipped last turn + const forceSearch = session.turnsSinceLastSearch >= 1; + if (forceSearch) { + const reminderText = session.language === "es" - ? "IMPORTANTE: DEBES usar search_biographical_context para buscar información relevante a la pregunta del usuario antes de responder." - : "IMPORTANT: You MUST use search_biographical_context to search for information relevant to the user's question before answering."; - messages.push({ - role: "system", - content: reminderText, - }); - } - - Monitor.debug("ConversationEngineService: LLM call", { - sessionId, - historyLength: session.history.length, - forceSearch, - }); - - const llmResult = + ? "IMPORTANTE: DEBES usar search_biographical_context para buscar información relevante a la pregunta del usuario antes de responder." + : "IMPORTANT: You MUST use search_biographical_context to search for information relevant to the user's question before answering."; + messages.push({ + role: "system", + content: reminderText, + }); + } + + Monitor.debug("ConversationEngineService: LLM call", { + sessionId, + historyLength: session.history.length, + forceSearch, + }); + + const llmResult = await OpenAIService.getInstance().chatWithToolResolution( - messages, - [SEARCH_TOOL], - async (toolCall: ToolCall) => { - return await this.resolveToolCall(session, toolCall); - }, - 2, - forceSearch ? "required" : "auto", + messages, + [SEARCH_TOOL], + async (toolCall: ToolCall) => { + return await this.resolveToolCall(session, toolCall); + }, + 2, + forceSearch ? "required" : "auto", ); - // Track whether search was used this turn - if (llmResult.resolvedToolCalls.length > 0) { - session.turnsSinceLastSearch = 0; - } else { - session.turnsSinceLastSearch++; - } - - const response = this.truncateIfTooLong(llmResult.response, 200); - - if (!response || response.trim().length === 0) { - Monitor.warning("ConversationEngineService: empty LLM response", { - sessionId, - }); - session.isProcessing = false; - return; - } - - Monitor.info("ConversationEngineService: agent response", { - sessionId, - responseLength: response.length, - }); - - // Add assistant response to history - session.history.push({ role: "assistant", content: response }); - - // 3. Text-to-Speech and send to client - await this.generateAndSendResponse(session, response); - } catch (err) { - Monitor.exception(err, "ConversationEngineService: pipeline failed"); - WsOrchestratorService.getInstance().broadcastToSession(sessionId, { - event: "agent-error", - message: "Something went wrong. Please try again.", - }); - } finally { - session.isProcessing = false; + // Track whether search was used this turn + if (llmResult.resolvedToolCalls.length > 0) { + session.turnsSinceLastSearch = 0; + } else { + session.turnsSinceLastSearch++; + } + + const response = this.truncateIfTooLong(llmResult.response, 200); + + if (!response || response.trim().length === 0) { + Monitor.warning("ConversationEngineService: empty LLM response", { + sessionId, + }); + session.isProcessing = false; + return; + } + + Monitor.info("ConversationEngineService: agent response", { + sessionId, + responseLength: response.length, + }); + + // Add assistant response to history + session.history.push({ role: "assistant", content: response }); + + // 3. Text-to-Speech and send to client + await this.generateAndSendResponse(session, response); + } catch (err) { + Monitor.exception(err, "ConversationEngineService: pipeline failed"); + WsOrchestratorService.getInstance().broadcastToSession(sessionId, { + event: "agent-error", + message: "Something went wrong. Please try again.", + }); + } finally { + session.isProcessing = false; + } } - } - /** + /** * Ends a conversation session and cleans up. * @param sessionId The session ID */ - public endConversation(sessionId: string): void { - if (this.sessions.has(sessionId)) { - this.sessions.delete(sessionId); - Monitor.info("ConversationEngineService: conversation ended", { - sessionId, - }); + public endConversation(sessionId: string): void { + if (this.sessions.has(sessionId)) { + this.sessions.delete(sessionId); + + Monitor.info("ConversationEngineService: conversation ended", { + sessionId, + }); + } } - } - /** + /** * Resolves a tool call from the LLM (Firecrawl search). * Also emits source-cited events to the frontend. */ - private async resolveToolCall( - session: ConversationSession, - toolCall: ToolCall, - ): Promise { - if (toolCall.name !== "search_biographical_context") { - return JSON.stringify({ error: "Unknown tool: " + toolCall.name }); - } + private async resolveToolCall( + session: ConversationSession, + toolCall: ToolCall, + ): Promise { + if (toolCall.name !== "search_biographical_context") { + return JSON.stringify({ error: "Unknown tool: " + toolCall.name }); + } + + const query = (toolCall.arguments.query as string) || ""; + const searchQuery = session.persona.name + " " + query; + + Monitor.info("ConversationEngineService: Firecrawl search", { + sessionId: session.sessionId, + query: searchQuery, + }); - const query = (toolCall.arguments.query as string) || ""; - const searchQuery = session.persona.name + " " + query; - - Monitor.info("ConversationEngineService: Firecrawl search", { - sessionId: session.sessionId, - query: searchQuery, - }); - - try { - const results = await FirecrawlService.getInstance().search( - searchQuery, - 3, - ); - - // Emit source cards to frontend - for (const result of results) { - session.sourceCounter++; - const sourceCard: SourceCardData = { - id: "src-" + session.sessionId + "-" + session.sourceCounter, - title: result.title, - url: result.url, - snippet: result.description || result.markdown.substring(0, 200), - agentId: session.persona.id, - timestamp: Date.now(), - }; - WsOrchestratorService.getInstance().emitSourceCited( - session.sessionId, - session.persona.id, - sourceCard, - ); - } - - // Format results for LLM context - return this.formatSearchResultsForLLM(results); - } catch (err) { - Monitor.exception( - err, - "ConversationEngineService: Firecrawl search failed", - ); - return JSON.stringify({ - error: "Search failed. Answer from your existing knowledge.", - }); + try { + const results = await FirecrawlService.getInstance().search( + searchQuery, + 3, + ); + + // Emit source cards to frontend + for (const result of results) { + session.sourceCounter++; + const sourceCard: SourceCardData = { + id: "src-" + session.sessionId + "-" + session.sourceCounter, + title: result.title, + url: result.url, + snippet: result.description || result.markdown.substring(0, 200), + agentId: session.persona.id, + timestamp: Date.now(), + }; + WsOrchestratorService.getInstance().emitSourceCited( + session.sessionId, + session.persona.id, + sourceCard, + ); + } + + // Format results for LLM context + return this.formatSearchResultsForLLM(results); + } catch (err) { + Monitor.exception( + err, + "ConversationEngineService: Firecrawl search failed", + ); + return JSON.stringify({ + error: "Search failed. Answer from your existing knowledge.", + }); + } } - } - /** + /** * Formats Firecrawl search results into a string the LLM can use. */ - private formatSearchResultsForLLM(results: FirecrawlSearchResult[]): string { - if (results.length === 0) { - return JSON.stringify({ - sources: [], - note: "No results found. Answer from your existing knowledge.", - }); - } + private formatSearchResultsForLLM(results: FirecrawlSearchResult[]): string { + if (results.length === 0) { + return JSON.stringify({ + sources: [], + note: "No results found. Answer from your existing knowledge.", + }); + } + + const sources = results.map((r, i) => { + return { + index: i + 1, + title: r.title, + url: r.url, + content: r.markdown.substring(0, 800), + }; + }); - const sources = results.map((r, i) => { - return { - index: i + 1, - title: r.title, - url: r.url, - content: r.markdown.substring(0, 800), - }; - }); - - return JSON.stringify({ - sources: sources, - instruction: + return JSON.stringify({ + sources: sources, + instruction: "Use these sources to inform your response. Reference them naturally, e.g., 'As documented in...' or 'According to historical records...'", - }); - } + }); + } - /** + /** * Generates TTS audio from text and sends it to the client via WebSocket. */ - private async generateAndSendResponse( - session: ConversationSession, - text: string, - ): Promise { - const voiceId = session.persona.voiceId; - - if (!voiceId) { - Monitor.warning( - "ConversationEngineService: no voiceId for persona, sending text only", - { - personaId: session.persona.id, - }, - ); - // Send text-only response if no voice configured - WsOrchestratorService.getInstance().emitAgentSpeaking( - session.sessionId, - session.persona.id, - "", // empty audio chunk - text, - ); - WsOrchestratorService.getInstance().emitAgentFinished( - session.sessionId, - session.persona.id, - ); - return; + private async generateAndSendResponse( + session: ConversationSession, + text: string, + ): Promise { + const voiceId = session.persona.voiceId; + + if (!voiceId) { + Monitor.warning( + "ConversationEngineService: no voiceId for persona, sending text only", + { + personaId: session.persona.id, + }, + ); + // Send text-only response if no voice configured + WsOrchestratorService.getInstance().emitAgentSpeaking( + session.sessionId, + session.persona.id, + "", // empty audio chunk + text, + ); + WsOrchestratorService.getInstance().emitAgentFinished( + session.sessionId, + session.persona.id, + ); + return; + } + + // Detect audio tag from response text (for frontend UI display) + const audioTag = this.detectAudioTag(text); + + // Clean text for TTS: convert [pause] to SSML , strip other emotion tags + const ttsText = this.prepareTextForTTS(text); + + // Generate TTS + const audioBuffer = await ElevenLabsService.getInstance().textToSpeech( + ttsText, + voiceId, + { + modelId: "eleven_multilingual_v2", + }, + ); + + // Send as a single base64 payload (not incremental TTS streaming yet) + const audioBase64 = audioBuffer.toString("base64"); + + // Send audio directly to client + WsOrchestratorService.getInstance().emitAgentSpeaking( + session.sessionId, + session.persona.id, + audioBase64, + text, + audioTag || undefined, + ); + + WsOrchestratorService.getInstance().emitAgentFinished( + session.sessionId, + session.persona.id, + ); } - // Detect audio tag from response text (for frontend UI display) - const audioTag = this.detectAudioTag(text); - - // Clean text for TTS: convert [pause] to SSML , strip other emotion tags - const ttsText = this.prepareTextForTTS(text); - - // Generate TTS - const audioBuffer = await ElevenLabsService.getInstance().textToSpeech( - ttsText, - voiceId, - { - modelId: "eleven_multilingual_v2", - }, - ); - - // Send as a single base64 payload (not incremental TTS streaming yet) - const audioBase64 = audioBuffer.toString("base64"); - - WsOrchestratorService.getInstance().emitAgentSpeaking( - session.sessionId, - session.persona.id, - audioBase64, - text, - audioTag || undefined, - ); - - WsOrchestratorService.getInstance().emitAgentFinished( - session.sessionId, - session.persona.id, - ); - } - - /** + /** * Detects the first audio tag in the response text. * @param text The LLM response text * @returns The detected audio tag (e.g., "angry", "excited") or null */ - private detectAudioTag(text: string): string | null { + private detectAudioTag(text: string): string | null { // Match any bracketed tag, then normalize known emotions (including Spanish variants) - const match = text.match(/\[([a-záéíóúñ]+)\]/i); - if (!match) return null; - const raw = match[1].toLowerCase(); - - // Normalize Spanish → English - const NORMALIZE: Record = { - pausa: "pause", - emocionado: "excited", - emocionada: "excited", - enojado: "angry", - enojada: "angry", - enfadado: "angry", - enfadada: "angry", - susurros: "whispers", - susurra: "whispers", - susurrando: "whispers", - melancolía: "melancholy", - melancólico: "melancholy", - melancólica: "melancholy", - triste: "sad", - sorprendido: "surprised", - sorprendida: "surprised", - pensativo: "thoughtful", - pensativa: "thoughtful", - riendo: "laughing", - risa: "laughing", - serio: "serious", - seria: "serious", - }; - return NORMALIZE[raw] || raw; - } - - /** + const match = text.match(/\[([a-záéíóúñ]+)\]/i); + if (!match) return null; + const raw = match[1].toLowerCase(); + + // Normalize Spanish → English + const NORMALIZE: Record = { + pausa: "pause", + emocionado: "excited", + emocionada: "excited", + enojado: "angry", + enojada: "angry", + enfadado: "angry", + enfadada: "angry", + susurros: "whispers", + susurra: "whispers", + susurrando: "whispers", + melancolía: "melancholy", + melancólico: "melancholy", + melancólica: "melancholy", + triste: "sad", + sorprendido: "surprised", + sorprendida: "surprised", + pensativo: "thoughtful", + pensativa: "thoughtful", + riendo: "laughing", + risa: "laughing", + serio: "serious", + seria: "serious", + }; + return NORMALIZE[raw] || raw; + } + + /** * Prepares text for TTS by replacing custom tags with TTS-compatible equivalents. * - [pause] → SSML (supported by eleven_multilingual_v2) * - Other emotion tags ([angry], [excited], [whispers], [melancholy]) are stripped @@ -543,60 +555,110 @@ export class ConversationEngineService { * @param text The LLM response text with custom tags * @returns Clean text ready for ElevenLabs TTS */ - private prepareTextForTTS(text: string): string { + private prepareTextForTTS(text: string): string { // Replace pause tags (English and Spanish) with SSML break - let cleaned = text.replace(/\[pause?\]/gi, ''); - cleaned = cleaned.replace(/\[pausa\]/gi, ''); - // Strip ALL remaining bracketed tags (any language) - cleaned = cleaned.replace(/\[[^\]]*\]/g, ""); - // Clean up any double spaces left behind - cleaned = cleaned.replace(/ +/g, " ").trim(); - return cleaned; - } - - /** + let cleaned = text.replace(/\[pause?\]/gi, ''); + cleaned = cleaned.replace(/\[pausa\]/gi, ''); + // Strip ALL remaining bracketed tags (any language) + cleaned = cleaned.replace(/\[[^\]]*\]/g, ""); + // Clean up any double spaces left behind + cleaned = cleaned.replace(/ +/g, " ").trim(); + return cleaned; + } + + /** * Truncates a response if it exceeds the word limit. * Cuts at the last sentence boundary within the limit. * @param text The response text * @param maxWords Maximum number of words allowed * @returns The truncated text */ - private truncateIfTooLong(text: string, maxWords: number): string { - if (!text) return text; - const words = text.split(/\s+/); - if (words.length <= maxWords) return text; - - Monitor.debug("ConversationEngineService: truncating response", { - originalWords: words.length, - maxWords: maxWords, - }); - - const truncated = words.slice(0, maxWords).join(" "); - // Find last sentence boundary - const lastPeriod = truncated.lastIndexOf("."); - const lastQuestion = truncated.lastIndexOf("?"); - const lastExclamation = truncated.lastIndexOf("!"); - const lastBoundary = Math.max(lastPeriod, lastQuestion, lastExclamation); - - if (lastBoundary > truncated.length * 0.5) { - return truncated.substring(0, lastBoundary + 1); + private truncateIfTooLong(text: string, maxWords: number): string { + if (!text) return text; + const words = text.split(/\s+/); + if (words.length <= maxWords) return text; + + Monitor.debug("ConversationEngineService: truncating response", { + originalWords: words.length, + maxWords: maxWords, + }); + + const truncated = words.slice(0, maxWords).join(" "); + // Find last sentence boundary + const lastPeriod = truncated.lastIndexOf("."); + const lastQuestion = truncated.lastIndexOf("?"); + const lastExclamation = truncated.lastIndexOf("!"); + const lastBoundary = Math.max(lastPeriod, lastQuestion, lastExclamation); + + if (lastBoundary > truncated.length * 0.5) { + return truncated.substring(0, lastBoundary + 1); + } + return truncated + "..."; } - return truncated + "..."; - } - /** + /** * Trims conversation history to the last N messages. * Always keeps the most recent messages for context. * @param history Full conversation history * @param maxMessages Maximum number of messages to keep */ - private trimHistory( - history: ChatMessage[], - maxMessages: number, - ): ChatMessage[] { - if (history.length <= maxMessages) { - return history; + private trimHistory( + history: ChatMessage[], + maxMessages: number, + ): ChatMessage[] { + if (history.length <= maxMessages) { + return history; + } + return history.slice(-maxMessages); + } + + /** + * Fetches an image URL and returns it as base64-encoded JPEG string. + * @param url The image URL to fetch + * @returns Base64-encoded image data, or null on failure + */ + private fetchImageAsBase64(url: string): Promise { + // Handle data URLs — already base64, just extract the payload + if (url.startsWith("data:")) { + const commaIdx = url.indexOf(","); + if (commaIdx > 0) { + return Promise.resolve(url.substring(commaIdx + 1)); + } + return Promise.resolve(null); + } + + return new Promise((resolve) => { + const client = url.startsWith("https") ? HTTPS : HTTP; + const req = client.get(url, { timeout: 15000 }, (res) => { + if (res.statusCode !== 200) { + Monitor.warning("ConversationEngineService: failed to fetch persona image", { + url: url.substring(0, 80), + statusCode: res.statusCode, + }); + resolve(null); + return; + } + + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => { + const buffer = Buffer.concat(chunks); + resolve(buffer.toString("base64")); + }); + res.on("error", () => resolve(null)); + }); + + req.on("error", (err) => { + Monitor.warning("ConversationEngineService: image fetch error", { + error: err.message, + }); + resolve(null); + }); + + req.on("timeout", () => { + req.destroy(); + resolve(null); + }); + }); } - return history.slice(-maxMessages); - } } diff --git a/web-application/backend/src/services/elevenlabs-service.ts b/web-application/backend/src/services/elevenlabs-service.ts index 666d67d..c98c52d 100644 --- a/web-application/backend/src/services/elevenlabs-service.ts +++ b/web-application/backend/src/services/elevenlabs-service.ts @@ -328,6 +328,171 @@ export class ElevenLabsService { * @param tools Array of tool definitions the agent can use * @returns ElevenAgentConfig ready to be sent to the Conversational AI API */ + /** + * Creates a new voice using ElevenLabs Voice Design (text-to-voice). + * Generates a voice from a natural language description. + * @param name Display name for the voice + * @param description Natural language description (e.g., "Male, 50s, Eastern European accent, formal") + * @returns The generated voiceId + */ + public async designVoice(name: string, description: string): Promise { + const config = ElevenLabsConfig.getInstance(); + const url = `${config.baseUrl}/v1/voice-generation/generate-voice`; + + Monitor.info("ElevenLabsService.designVoice", { name, descLength: description.length }); + + return new Promise((resolve, reject) => { + const body = JSON.stringify({ + voice_description: description, + text: `Hello, I am ${name}. It is a pleasure to make your acquaintance. I have much to share with you about my life and times.`, + }); + + const urlObj = new URL(url); + const req = HTTPS.request({ + hostname: urlObj.hostname, + port: 443, + path: urlObj.pathname, + method: "POST", + headers: { + "xi-api-key": config.apiKey, + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }, + }, (response) => { + let responseData = ""; + response.on("data", (chunk) => { responseData += chunk; }); + response.on("end", () => { + if (response.statusCode !== 200) { + Monitor.warning("ElevenLabsService.designVoice API error", { + statusCode: response.statusCode, + error: responseData.substring(0, 200), + }); + return reject(Object.assign( + new Error("Voice Design failed (status " + response.statusCode + ")"), + { code: "ELEVENLABS_ERROR" }, + )); + } + + try { + const parsed = JSON.parse(responseData); + const voiceId = parsed.voice_id || ""; + if (!voiceId) { + return reject(Object.assign( + new Error("Voice Design returned no voice_id"), + { code: "ELEVENLABS_ERROR" }, + )); + } + Monitor.info("ElevenLabsService.designVoice completed", { voiceId }); + resolve(voiceId); + } catch (ex) { + Monitor.exception(ex, "ElevenLabsService.designVoice parse failed"); + reject(Object.assign(new Error("Failed to parse Voice Design response"), { code: "ELEVENLABS_ERROR" })); + } + }); + }); + + req.on("error", (err) => { + Monitor.exception(err, "ElevenLabsService.designVoice request failed"); + reject(Object.assign(new Error("Voice Design request failed: " + err.message), { code: "ELEVENLABS_ERROR" })); + }); + + req.write(body); + req.end(); + }); + } + + /** + * Clones a voice using ElevenLabs Instant Voice Cloning (IVC). + * @param name Display name for the cloned voice + * @param audioBuffer Audio sample buffer (MP3/WAV, 1-2 min) + * @returns The cloned voiceId + */ + public async cloneVoice(name: string, audioBuffer: Buffer): Promise { + const config = ElevenLabsConfig.getInstance(); + const url = `${config.baseUrl}/v1/voices/add`; + + Monitor.info("ElevenLabsService.cloneVoice", { name, audioBytes: audioBuffer.length }); + + return new Promise((resolve, reject) => { + const boundary = "----FormBoundary" + Date.now().toString(16); + const parts: Buffer[] = []; + + // Name field + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="name"\r\n\r\n${name}\r\n`, + )); + + // Audio file + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="files"; filename="sample.mp3"\r\nContent-Type: audio/mpeg\r\n\r\n`, + )); + parts.push(audioBuffer); + parts.push(Buffer.from("\r\n")); + + // Remove background noise + parts.push(Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="remove_background_noise"\r\n\r\ntrue\r\n`, + )); + + // End boundary + parts.push(Buffer.from(`--${boundary}--\r\n`)); + + const body = Buffer.concat(parts); + + const urlObj = new URL(url); + const req = HTTPS.request({ + hostname: urlObj.hostname, + port: 443, + path: urlObj.pathname, + method: "POST", + headers: { + "xi-api-key": config.apiKey, + "Content-Type": "multipart/form-data; boundary=" + boundary, + "Content-Length": body.length, + }, + }, (response) => { + let responseData = ""; + response.on("data", (chunk) => { responseData += chunk; }); + response.on("end", () => { + if (response.statusCode !== 200) { + Monitor.warning("ElevenLabsService.cloneVoice API error", { + statusCode: response.statusCode, + error: responseData.substring(0, 200), + }); + return reject(Object.assign( + new Error("Voice Clone failed (status " + response.statusCode + ")"), + { code: "ELEVENLABS_ERROR" }, + )); + } + + try { + const parsed = JSON.parse(responseData); + const voiceId = parsed.voice_id || ""; + if (!voiceId) { + return reject(Object.assign( + new Error("Voice Clone returned no voice_id"), + { code: "ELEVENLABS_ERROR" }, + )); + } + Monitor.info("ElevenLabsService.cloneVoice completed", { voiceId }); + resolve(voiceId); + } catch (ex) { + Monitor.exception(ex, "ElevenLabsService.cloneVoice parse failed"); + reject(Object.assign(new Error("Failed to parse Voice Clone response"), { code: "ELEVENLABS_ERROR" })); + } + }); + }); + + req.on("error", (err) => { + Monitor.exception(err, "ElevenLabsService.cloneVoice request failed"); + reject(Object.assign(new Error("Voice Clone request failed: " + err.message), { code: "ELEVENLABS_ERROR" })); + }); + + req.write(body); + req.end(); + }); + } + public getAgentConfig(persona: string, voiceId: string, tools: ElevenAgentTool[]): ElevenAgentConfig { return { agent: { diff --git a/web-application/backend/src/services/image-generation-service.ts b/web-application/backend/src/services/image-generation-service.ts new file mode 100644 index 0000000..a63d665 --- /dev/null +++ b/web-application/backend/src/services/image-generation-service.ts @@ -0,0 +1,182 @@ +// Image generation service — Google Imagen / DALL-E for character portraits + +"use strict"; + +import HTTPS from "https"; +import { Monitor } from "../monitor"; +import { ImageGenerationConfig } from "../config/config-image-generation"; + +/** + * Generates historical portrait images using Google Imagen or DALL-E. + * Used during custom character creation to produce persona portraits. + */ +export class ImageGenerationService { + /* Singleton */ + + public static instance: ImageGenerationService = null; + + public static getInstance(): ImageGenerationService { + if (ImageGenerationService.instance) { + return ImageGenerationService.instance; + } else { + ImageGenerationService.instance = new ImageGenerationService(); + return ImageGenerationService.instance; + } + } + + constructor() {} + + /** + * Checks if image generation is available. + */ + public isEnabled(): boolean { + return ImageGenerationConfig.getInstance().enabled; + } + + /** + * Generates a portrait image from a text prompt. + * @param prompt Detailed image generation prompt + * @returns Base64-encoded image data, or null on failure + */ + public async generatePortrait(prompt: string): Promise { + const config = ImageGenerationConfig.getInstance(); + + if (!config.enabled) { + Monitor.warning("ImageGenerationService: no provider configured"); + return null; + } + + if (config.provider === "imagen") { + return this.generateWithImagen(prompt); + } else if (config.provider === "dalle") { + return this.generateWithDalle(prompt); + } + + return null; + } + + /** + * Generates an image using Google Imagen API. + */ + private async generateWithImagen(prompt: string): Promise { + const config = ImageGenerationConfig.getInstance(); + const enhancedPrompt = `Historical portrait painting, photorealistic style, dramatic chiaroscuro lighting, museum quality. ${prompt}. Dark moody background, slight sepia tone, 3/4 view portrait composition.`; + + try { + const body = JSON.stringify({ + instances: [{ prompt: enhancedPrompt }], + parameters: { + sampleCount: 1, + aspectRatio: "1:1", + outputMimeType: "image/jpeg", + }, + }); + + const url = `https://generativelanguage.googleapis.com/v1beta/models/${config.imagenModel}:predict?key=${config.googleApiKey}`; + + const response = await this.httpsPost(url, body, { + "Content-Type": "application/json", + }); + + const parsed = JSON.parse(response); + if (parsed.predictions && parsed.predictions[0] && parsed.predictions[0].bytesBase64Encoded) { + Monitor.info("ImageGenerationService: Imagen portrait generated"); + return parsed.predictions[0].bytesBase64Encoded; + } + + Monitor.warning("ImageGenerationService: Imagen returned no image", { + response: response.substring(0, 200), + }); + return null; + } catch (err) { + Monitor.exception(err, "ImageGenerationService: Imagen generation failed"); + // Fallback to DALL-E if available + if (config.openaiApiKey) { + Monitor.info("ImageGenerationService: falling back to DALL-E"); + return this.generateWithDalle(prompt); + } + return null; + } + } + + /** + * Generates an image using OpenAI DALL-E API. + */ + private async generateWithDalle(prompt: string): Promise { + const config = ImageGenerationConfig.getInstance(); + const enhancedPrompt = `Historical portrait painting, photorealistic, dramatic lighting, museum quality. ${prompt}. Dark moody background, slight sepia tone.`; + + try { + const body = JSON.stringify({ + model: "dall-e-3", + prompt: enhancedPrompt, + n: 1, + size: "1024x1024", + response_format: "b64_json", + }); + + const response = await this.httpsPost( + "https://api.openai.com/v1/images/generations", + body, + { + "Content-Type": "application/json", + "Authorization": "Bearer " + config.openaiApiKey, + }, + ); + + const parsed = JSON.parse(response); + if (parsed.data && parsed.data[0] && parsed.data[0].b64_json) { + Monitor.info("ImageGenerationService: DALL-E portrait generated"); + return parsed.data[0].b64_json; + } + + Monitor.warning("ImageGenerationService: DALL-E returned no image"); + return null; + } catch (err) { + Monitor.exception(err, "ImageGenerationService: DALL-E generation failed"); + return null; + } + } + + /** + * Simple HTTPS POST helper that returns the response body as a string. + */ + private httpsPost(url: string, body: string, headers: Record): Promise { + return new Promise((resolve, reject) => { + const urlObj = new URL(url); + const options = { + hostname: urlObj.hostname, + port: 443, + path: urlObj.pathname + urlObj.search, + method: "POST", + headers: { + ...headers, + "Content-Length": Buffer.byteLength(body), + }, + timeout: 60000, + }; + + const req = HTTPS.request(options, (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => { + const responseBody = Buffer.concat(chunks).toString(); + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + resolve(responseBody); + } else { + reject(new Error(`HTTP ${res.statusCode}: ${responseBody.substring(0, 300)}`)); + } + }); + }); + + req.on("error", reject); + req.on("timeout", () => { + req.destroy(); + reject(new Error("Request timeout")); + }); + + req.write(body); + req.end(); + }); + } +} diff --git a/web-application/backend/src/services/voice-discovery-service.ts b/web-application/backend/src/services/voice-discovery-service.ts new file mode 100644 index 0000000..bc3634c --- /dev/null +++ b/web-application/backend/src/services/voice-discovery-service.ts @@ -0,0 +1,198 @@ +// Voice discovery service — searches for real audio recordings via Firecrawl + +"use strict"; + +import HTTP from "http"; +import HTTPS from "https"; +import { Monitor } from "../monitor"; +import { FirecrawlService } from "./firecrawl-service"; + +/** + * Result of a voice discovery search + */ +export interface VoiceDiscoveryResult { + type: "real" | "generated"; + audioUrl?: string; + audioSource?: string; + voiceDescription?: string; +} + +/** + * Searches web audio archives for real voice recordings of historical figures. + * Uses Firecrawl to search archive.org, Library of Congress, BBC, and other sources. + * If found, the audio can be used for ElevenLabs IVC (Instant Voice Cloning). + */ +export class VoiceDiscoveryService { + /* Singleton */ + + public static instance: VoiceDiscoveryService = null; + + public static getInstance(): VoiceDiscoveryService { + if (VoiceDiscoveryService.instance) { + return VoiceDiscoveryService.instance; + } else { + VoiceDiscoveryService.instance = new VoiceDiscoveryService(); + return VoiceDiscoveryService.instance; + } + } + + constructor() {} + + /** + * Searches for real audio recordings of a historical figure. + * Queries audio archive sites via Firecrawl, parses results for audio URLs. + * @param name The historical figure's name + * @param voiceDescription Fallback description for Voice Design + * @returns Discovery result with audio URL or fallback description + */ + public async findVoiceReference(name: string, voiceDescription: string): Promise { + Monitor.info("VoiceDiscoveryService: searching for voice", { name }); + + const audioUrls: Array<{ url: string; source: string }> = []; + + // Search 1: archive.org + try { + const results = await FirecrawlService.getInstance().search( + `"${name}" speech recording audio site:archive.org`, + 3, + ); + for (const r of results) { + const urls = this.extractAudioUrls(r.markdown + " " + r.url); + for (const u of urls) { + audioUrls.push({ url: u, source: "archive.org" }); + } + } + } catch (err) { + Monitor.warning("VoiceDiscoveryService: archive.org search failed", { error: (err as Error).message }); + } + + await this.delay(12000); // Rate limiting + + // Search 2: Library of Congress + try { + const results = await FirecrawlService.getInstance().search( + `"${name}" voice recording audio site:loc.gov`, + 3, + ); + for (const r of results) { + const urls = this.extractAudioUrls(r.markdown + " " + r.url); + for (const u of urls) { + audioUrls.push({ url: u, source: "Library of Congress" }); + } + } + } catch (err) { + Monitor.warning("VoiceDiscoveryService: LoC search failed", { error: (err as Error).message }); + } + + await this.delay(12000); + + // Search 3: General audio search + try { + const results = await FirecrawlService.getInstance().search( + `"${name}" historical speech audio mp3 recording`, + 3, + ); + for (const r of results) { + const urls = this.extractAudioUrls(r.markdown + " " + r.url); + for (const u of urls) { + audioUrls.push({ url: u, source: r.title }); + } + } + } catch (err) { + Monitor.warning("VoiceDiscoveryService: general audio search failed", { error: (err as Error).message }); + } + + for (const candidate of audioUrls) { + const isValid = await this.validateAudioUrl(candidate.url); + if (isValid) { + Monitor.info("VoiceDiscoveryService: found real audio", { + name, + url: candidate.url, + source: candidate.source, + }); + return { + type: "real", + audioUrl: candidate.url, + audioSource: candidate.source, + }; + } + Monitor.info("VoiceDiscoveryService: audio URL invalid, trying next", { url: candidate.url }); + } + + Monitor.info("VoiceDiscoveryService: no real audio found, using Voice Design", { name }); + return { + type: "generated", + voiceDescription: voiceDescription, + }; + } + + /** + * Downloads audio from a URL and returns the buffer. + * @param url The audio URL to download + * @returns Audio buffer or null on failure + */ + public async downloadAudio(url: string): Promise { + return new Promise((resolve) => { + const client = url.startsWith("https") ? HTTPS : HTTP; + client.get(url, { timeout: 30000 }, (res) => { + if (res.statusCode !== 200) { + resolve(null); + return; + } + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => resolve(Buffer.concat(chunks))); + res.on("error", () => resolve(null)); + }).on("error", () => resolve(null)); + }); + } + + /** + * Extracts audio file URLs from text content. + */ + private extractAudioUrls(text: string): string[] { + const urls: string[] = []; + // Match URLs ending in audio extensions + const regex = /https?:\/\/[^\s"'<>)]+\.(?:mp3|wav|ogg|flac|m4a)/gi; + const matches = text.match(regex); + if (matches) { + for (const m of matches) { + if (!urls.includes(m)) { + urls.push(m); + } + } + } + return urls; + } + + /** + * Validates an audio URL by making a HEAD request. + */ + private validateAudioUrl(url: string): Promise { + return new Promise((resolve) => { + try { + const urlObj = new URL(url); + const options = { + hostname: urlObj.hostname, + path: urlObj.pathname + urlObj.search, + method: "HEAD", + timeout: 10000, + }; + + const client = url.startsWith("https") ? HTTPS : require("http"); + const req = client.request(options, (res: any) => { + resolve(res.statusCode === 200); + }); + req.on("error", () => resolve(false)); + req.on("timeout", () => { req.destroy(); resolve(false); }); + req.end(); + } catch { + resolve(false); + } + }); + } + + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} diff --git a/web-application/backend/src/services/ws-orchestrator-service.ts b/web-application/backend/src/services/ws-orchestrator-service.ts index 2ba26ae..2b7954b 100644 --- a/web-application/backend/src/services/ws-orchestrator-service.ts +++ b/web-application/backend/src/services/ws-orchestrator-service.ts @@ -4,7 +4,9 @@ import { Monitor } from "../monitor"; import { WebsocketController } from "../controllers/websocket/websocket"; -import { SourceCardData, AgentSpeakingEvent, SourceCitedEvent, AgentFinishedEvent, SessionEndEvent } from "../models/ws-events"; +import { + SourceCardData, AgentSpeakingEvent, SourceCitedEvent, AgentFinishedEvent, SessionEndEvent, +} from "../models/ws-events"; /** * Active WebSocket session @@ -175,4 +177,5 @@ export class WsOrchestratorService { this.broadcastToSession(sessionId, event); this.removeSession(sessionId); } + } diff --git a/web-application/frontend/components/session/BackgroundEffect.vue b/web-application/frontend/components/session/BackgroundEffect.vue new file mode 100644 index 0000000..29a0707 --- /dev/null +++ b/web-application/frontend/components/session/BackgroundEffect.vue @@ -0,0 +1,364 @@ + + + diff --git a/web-application/frontend/components/session/SessionPortrait.vue b/web-application/frontend/components/session/SessionPortrait.vue index 10beabd..02980ef 100644 --- a/web-application/frontend/components/session/SessionPortrait.vue +++ b/web-application/frontend/components/session/SessionPortrait.vue @@ -31,11 +31,12 @@ /> - +
+ (""); const userTranscriptText = ref(""); const agentErrorMessage = ref(""); - // Parse incoming messages watch(data, (raw) => { if (!raw) return; @@ -137,6 +136,7 @@ export function useOrchestratorSocket() { case "agent-error": agentErrorMessage.value = (msg as any).message || "Unknown error"; break; + } }); diff --git a/web-application/frontend/i18n/locales/en.json b/web-application/frontend/i18n/locales/en.json index 9bd2f70..b43d8f3 100644 --- a/web-application/frontend/i18n/locales/en.json +++ b/web-application/frontend/i18n/locales/en.json @@ -1,7 +1,9 @@ { + "{count} Spirits Summoned Today": "{count} Spirits Summoned Today", "2FA successfully disabled": "", "2FA successfully enabled": "", "A verification link has been sent to your email. Once verified, you can log in.": "", + "AI-generated voice": "", "About": "", "Access": "", "Access denied. You lack the required permission to visit this page.": "", @@ -37,6 +39,8 @@ "Back to homepage": "", "Ban users": "", "Banned": "", + "Begin the Seance": "", + "Begin the Summoning": "", "Begin the Séance": "", "Both passwords must match.": "", "Browser": "", @@ -59,11 +63,12 @@ "Copy Quote": "", "Copy caption for Instagram/TikTok": "", "Could not connect to the server": "", - "DeadTalk - Talk to History": "", "Could not load the user details": "", "Could not load the user role": "", "Could not load user activation state because the user does not exist.": "", "Could not load user state": "", + "Create Character — DeadTalk": "", + "Create Your Own": "", "Create account": "", "Created at": "", "Current password": "", @@ -74,6 +79,7 @@ "Default role": "", "Default settings": "", "Default theme": "", + "DeadTalk - Talk to History": "DeadTalk - Talk to History", "Delete": "", "Delete account": "", "Delete image": "", @@ -93,6 +99,7 @@ "Email verification state": "", "Enable 2FA authentication": "", "Enable account": "", + "Enter a name and we'll research, paint, and give voice to any figure from history": "", "Enter a secret key": "", "Enter your code": "", "Enter your password": "", @@ -124,6 +131,8 @@ "Grants the ability to list and moderate users on the platform.": "", "Grants the permission to ban users from the platform.": "", "Have real voice conversations with history's greatest minds.": "", + "Hints (optional)": "", + "Historical Figure": "", "Home": "", "ID": "", "IP Address": "", @@ -235,7 +244,6 @@ "Provides immunity from moderation actions.": "", "Ready": "", "Recover": "", - "{count} Spirits Summoned Today": "{count} Spirits Summoned Today", "Recover password": "", "Remote device session removed": "", "Remove": "", @@ -288,7 +296,11 @@ "Start fresh": "", "State": "", "Submitting": "", + "Summon a New Spirit": "", + "Summon another spirit": "", + "Summon any historical figure": "", "Summoned from": "", + "Summoning": "", "Summoning spirits...": "", "Switch to auto": "", "Switch to push-to-talk": "", @@ -351,6 +363,7 @@ "Verified": "", "Verified Artifact": "", "View details": "", + "Voice cloned from real recording": "", "We did not find any account with the provided email": "", "Website": "", "We’ve sent a recovery link to your email address. Follow the instructions to reset your password.": "", @@ -365,6 +378,8 @@ "[m] [d], [y]": "", "[m] [d], [y] [hh]:[mm]:[ss]": "", "current": "", + "e.g., Italian Renaissance artist, inventor": "", + "e.g., Leonardo da Vinci": "", "of": "", "result": "", "results": "", diff --git a/web-application/frontend/i18n/locales/es.json b/web-application/frontend/i18n/locales/es.json index a18f145..bf97166 100644 --- a/web-application/frontend/i18n/locales/es.json +++ b/web-application/frontend/i18n/locales/es.json @@ -1,7 +1,9 @@ { + "{count} Spirits Summoned Today": "{count} espíritus invocados hoy", "2FA successfully disabled": "2FA desactivado correctamente", "2FA successfully enabled": "2FA activado correctamente", "A verification link has been sent to your email. Once verified, you can log in.": "Un enlace de verificación ha sido enviado a tu correo electrónico. Una vez verificado, podrás iniciar sesión.", + "AI-generated voice": "Voz generada por IA", "About": "Acerca de", "Access": "Acceder", "Access denied. You lack the required permission to visit this page.": "Acceso denegado. No tienes el permiso requerido para visitar esta página.", @@ -37,6 +39,8 @@ "Back to homepage": "Volver a inicio", "Ban users": "Bloquear usuarios", "Banned": "Expulsado", + "Begin the Seance": "Iniciar la Séance", + "Begin the Summoning": "Iniciar la Invocación", "Begin the Séance": "Comienza la sesión", "Both passwords must match.": "Ambas contraseñas deben coincidir.", "Browser": "Navegador", @@ -59,11 +63,12 @@ "Copy Quote": "Copiar cita", "Copy caption for Instagram/TikTok": "Copiar texto para Instagram/TikTok", "Could not connect to the server": "No se pudo conectar al servidor", - "DeadTalk - Talk to History": "DeadTalk - Habla con la Historia", "Could not load the user details": "No se pudieron cargar los detalles del usuario.", "Could not load the user role": "No se pudo cargar el rol del usuario.", "Could not load user activation state because the user does not exist.": "No se pudo cargar el estado de activación del usuario porque no existe", "Could not load user state": "No se pudo cargar el estado del usuario", + "Create Character — DeadTalk": "Crear Personaje — DeadTalk", + "Create Your Own": "Crea el Tuyo", "Create account": "Crear cuenta", "Created at": "Fecha de creación", "Current password": "Contraseña actual", @@ -74,6 +79,7 @@ "Default role": "Rol predeterminado ", "Default settings": "Ajustes predeterminados", "Default theme": "Tema predeterminado", + "DeadTalk - Talk to History": "DeadTalk - Habla con la Historia", "Delete": "Eliminar", "Delete account": "Eliminar cuenta", "Delete image": "Eliminar imagen", @@ -93,6 +99,7 @@ "Email verification state": "Estado de verificación de correo electrónico", "Enable 2FA authentication": "Habilitar autenticación de 2FA", "Enable account": "Habilitar cuenta", + "Enter a name and we'll research, paint, and give voice to any figure from history": "Introduce un nombre y investigaremos, pintaremos y daremos voz a cualquier figura de la historia", "Enter a secret key": "Introduce una clave secreta", "Enter your code": "Introduce tu código", "Enter your password": "Introduce tu contraseña", @@ -124,6 +131,8 @@ "Grants the ability to list and moderate users on the platform.": "Concede la capacidad para listar y moderar usuarios en la plataforma.", "Grants the permission to ban users from the platform.": "Concede el permiso para bloquear usuarios de la plataforma.", "Have real voice conversations with history's greatest minds.": "Mantén conversaciones de voz reales con las mentes más brillantes de la historia.", + "Hints (optional)": "Pistas (opcional)", + "Historical Figure": "Figura Histórica", "Home": "Inicio", "ID": "ID", "IP Address": "Dirección IP", @@ -235,7 +244,6 @@ "Provides immunity from moderation actions.": "Proporciona inmunidad contra acciones de moderación.", "Ready": "Listo", "Recover": "Recuperar", - "{count} Spirits Summoned Today": "{count} espíritus invocados hoy", "Recover password": "Recuperar contraseña", "Remote device session removed": "Sesión de dispositivo remoto eliminada", "Remove": "Eliminar", @@ -288,7 +296,11 @@ "Start fresh": "Empezar de cero", "State": "Estado", "Submitting": "Enviando", + "Summon a New Spirit": "Invocar un Nuevo Espíritu", + "Summon another spirit": "Invocar otro espíritu", + "Summon any historical figure": "Invoca cualquier figura histórica", "Summoned from": "Invocado desde", + "Summoning": "Invocando", "Summoning spirits...": "Invocando espíritus...", "Switch to auto": "Cambiar a automático", "Switch to push-to-talk": "Cambiar a pulsar para hablar", @@ -351,6 +363,7 @@ "Verified": "Verificado", "Verified Artifact": "Artefacto verificado", "View details": "Ver detalles", + "Voice cloned from real recording": "Voz clonada de grabación real", "We did not find any account with the provided email": "No encontramos ninguna cuenta con el correo proporcionado", "Website": "Sitio web", "We’ve sent a recovery link to your email address. Follow the instructions to reset your password.": "Hemos enviado un enlace de recuperación a tu correo electrónico. Sigue las instrucciones para restablecer tu contraseña.", @@ -365,6 +378,8 @@ "[m] [d], [y]": "[d] de [m], [y]", "[m] [d], [y] [hh]:[mm]:[ss]": "[d] de [m], [y] [hh]:[mm]:[ss]", "current": "actual", + "e.g., Italian Renaissance artist, inventor": "ej., artista del Renacimiento italiano, inventor", + "e.g., Leonardo da Vinci": "ej., Leonardo da Vinci", "of": "de", "result": "resultado", "results": "resultados", diff --git a/web-application/frontend/pages/create-character.vue b/web-application/frontend/pages/create-character.vue new file mode 100644 index 0000000..b2ac783 --- /dev/null +++ b/web-application/frontend/pages/create-character.vue @@ -0,0 +1,331 @@ + + + diff --git a/web-application/frontend/pages/index.vue b/web-application/frontend/pages/index.vue index 29ba0b0..20febb2 100644 --- a/web-application/frontend/pages/index.vue +++ b/web-application/frontend/pages/index.vue @@ -1,5 +1,57 @@