From c36e8ba519e5ba2eb5c89d6ce8ac2af5d129f6ed Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Wed, 10 Jun 2026 15:22:34 -0700 Subject: [PATCH 01/40] add github.com/makiuchi-d/gozxing dependency for go to scan qr code --- go.mod | 3 +++ go.sum | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/go.mod b/go.mod index 9ee39fcdea..8e9579ec11 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gordonklaus/portaudio v0.0.0-20260203164431-765aa7dfa631 github.com/gorilla/websocket v1.5.3 + github.com/makiuchi-d/gozxing v0.1.1 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 go.uber.org/zap v1.27.0 @@ -32,6 +33,8 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e7ed64a3c5..e86e18b043 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/makiuchi-d/gozxing v0.1.1 h1:xxqijhoedi+/lZlhINteGbywIrewVdVv2wl9r5O9S1I= +github.com/makiuchi-d/gozxing v0.1.1/go.mod h1:eRIHbOjX7QWxLIDJoQuMLhuXg9LAuw6znsUtRkNw9DU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -62,6 +64,10 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e042bb1ed437bb2107910085a342aeb4f80f1168 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Wed, 10 Jun 2026 16:25:38 -0700 Subject: [PATCH 02/40] add luma integration --- config/conversation.json5 | 2 +- config/luma_checkin.json5 | 73 +++++ config/luma_checkin_local.json5 | 38 +++ plugins/actions/actions.go | 1 + plugins/actions/luma_checkin/export_test.go | 47 +++ plugins/actions/luma_checkin/luma_checkin.go | 291 ++++++++++++++++++ .../actions/luma_checkin/luma_checkin_test.go | 196 ++++++++++++ plugins/actions/luma_checkin/luma_client.go | 198 ++++++++++++ .../actions/luma_checkin/luma_client_test.go | 155 ++++++++++ plugins/inputs/inputs.go | 1 + plugins/inputs/qr_scanner/debounce.go | 47 +++ plugins/inputs/qr_scanner/debounce_test.go | 62 ++++ plugins/inputs/qr_scanner/decode.go | 34 ++ plugins/inputs/qr_scanner/decode_test.go | 73 +++++ plugins/inputs/qr_scanner/export_test.go | 16 + plugins/inputs/qr_scanner/qr_scanner.go | 243 +++++++++++++++ plugins/inputs/qr_scanner/url_parse.go | 32 ++ plugins/inputs/qr_scanner/url_parse_test.go | 118 +++++++ 18 files changed, 1626 insertions(+), 1 deletion(-) create mode 100644 config/luma_checkin.json5 create mode 100644 config/luma_checkin_local.json5 create mode 100644 plugins/actions/luma_checkin/export_test.go create mode 100644 plugins/actions/luma_checkin/luma_checkin.go create mode 100644 plugins/actions/luma_checkin/luma_checkin_test.go create mode 100644 plugins/actions/luma_checkin/luma_client.go create mode 100644 plugins/actions/luma_checkin/luma_client_test.go create mode 100644 plugins/inputs/qr_scanner/debounce.go create mode 100644 plugins/inputs/qr_scanner/debounce_test.go create mode 100644 plugins/inputs/qr_scanner/decode.go create mode 100644 plugins/inputs/qr_scanner/decode_test.go create mode 100644 plugins/inputs/qr_scanner/export_test.go create mode 100644 plugins/inputs/qr_scanner/qr_scanner.go create mode 100644 plugins/inputs/qr_scanner/url_parse.go create mode 100644 plugins/inputs/qr_scanner/url_parse_test.go diff --git a/config/conversation.json5 b/config/conversation.json5 index aaa14a50c1..a412c3a259 100644 --- a/config/conversation.json5 +++ b/config/conversation.json5 @@ -21,7 +21,7 @@ }, }, { - type: "VLMGemini", + type: "VLMVila", } ], cortex_llm: { diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 new file mode 100644 index 0000000000..d73c98042a --- /dev/null +++ b/config/luma_checkin.json5 @@ -0,0 +1,73 @@ +{ + // Luma event check-in: scans QR codes via camera, looks guests up via Luma's + // external check-in API, speaks a personalized welcome, and (via the existing + // emotion + unitree_g1_arm plugins) shows a happy face and waves. + // + // Required env vars: + // LUMA_API_KEY - Luma API key with external-check-in scope + // LUMA_EVENT_ID - api_id of the event to check guests into (e.g. evt-...) + // ELEVENLABS_API_KEY - ElevenLabs TTS API key + version: "v1.0.0", + hertz: 0.2, + name: "luma_checkin", + api_key: "${OM_API_KEY:-openmind_free}", + system_prompt_base: "You are a friendly event greeter robot stationed at a Luma event check-in desk.\ +\n\ +When you receive a 'qr_scan: pk=...' input from the QR scanner, respond in a single turn with three actions, all called concurrently:\n\ + 1. luma_checkin with that pk (this looks up the guest and speaks the welcome).\n\ + 2. emotion with action 'happy'.\n\ + 3. unitree_g1_arm with action 'face_wave'.\n\ +\n\ +After luma_checkin returns, you may optionally make one short follow-up comment. Treat each scan as exactly one check-in (the input layer already debounces).", + system_governance: "Never check the same guest in twice within the same scan window. Never speak before luma_checkin returns.", + agent_inputs: [ + { + type: "QRScanner", + config: { + camera_index: 0, + capture_fps: 15, + decode_fps: 5, + resolution_width: 640, + resolution_height: 480, + jpeg_quality: 60, + dedupe_window_seconds: 30, + }, + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 6, + }, + }, + action_execution_mode: "concurrent", + agent_actions: [ + { + name: "luma_checkin", + llm_label: "luma_checkin", + connector: "api", + config: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Glad you made it.", + request_timeout_seconds: 5, + tts: { + provider: "elevenlabs", + elevenlabs_api_key: "${ELEVENLABS_API_KEY}", + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + }, + { + name: "emotion", + llm_label: "emotion", + connector: "zenoh", + }, + { + name: "unitree_g1_arm", + llm_label: "unitree_g1_arm", + connector: "zenoh", + }, + ], +} diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 new file mode 100644 index 0000000000..3c17bcaf93 --- /dev/null +++ b/config/luma_checkin_local.json5 @@ -0,0 +1,38 @@ +{ +version: "v1.0.0", + hertz: 0.2, + name: "luma_checkin_local", + api_key: "${OM_API_KEY:-openmind_free}", + system_prompt_base: "You are a test harness for the Luma check-in flow. The Vision input emits a line of the form 'qr_scan: pk= event=' when it sees a Luma check-in QR code, or 'none' when it does not. When you see 'qr_scan: pk=...', immediately call the luma_checkin action with that pk value. When you see 'none', do nothing. Do not narrate.", + system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", + agent_inputs: [ + { + type: "VLMVila", + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 4, + }, + }, + agent_actions: [ + { + name: "luma_checkin", + llm_label: "luma_checkin", + connector: "api", + config: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + request_timeout_seconds: 5, + tts: { + provider: "elevenlabs", + elevenlabs_api_key: "${ELEVENLABS_API_KEY}", + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + }, + ], +} diff --git a/plugins/actions/actions.go b/plugins/actions/actions.go index ef887be56f..4809e14d75 100644 --- a/plugins/actions/actions.go +++ b/plugins/actions/actions.go @@ -3,6 +3,7 @@ package actions import ( _ "github.com/openmind/om1/plugins/actions/emotion" _ "github.com/openmind/om1/plugins/actions/greeting_conversation" + _ "github.com/openmind/om1/plugins/actions/luma_checkin" _ "github.com/openmind/om1/plugins/actions/navigation" _ "github.com/openmind/om1/plugins/actions/speak" _ "github.com/openmind/om1/plugins/actions/unitree/g1/arm" diff --git a/plugins/actions/luma_checkin/export_test.go b/plugins/actions/luma_checkin/export_test.go new file mode 100644 index 0000000000..4697e74ee0 --- /dev/null +++ b/plugins/actions/luma_checkin/export_test.go @@ -0,0 +1,47 @@ +package luma_checkin + +import ( + "context" + "time" + + "go.uber.org/zap" +) + +const ( + GetGuestPath = getGuestPath + UpdateGuestStatusPath = updateGuestStatusPath + DefaultGreetingTemplate = defaultGreetingTemplate + DefaultStatus = defaultStatus +) + +var ( + ErrLumaNotFound = errLumaNotFound + ErrLumaUnauthorized = errLumaUnauthorized +) + +type LumaClient = lumaClient + +func NewLumaClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *LumaClient { + return newLumaClient(baseURL, apiKey, eventAPIID, timeout) +} + +func FirstNameFor(g *Guest) string { return firstNameFor(g) } +func FormatGreeting(tmpl string, g *Guest) string { return formatGreeting(tmpl, g) } + +type GuestLookup = guestLookup +type TTSPlayer = ttsPlayer + +type TestConnector struct{ *connector } + +func NewTestConnector(client GuestLookup, cfg Config, tts TTSPlayer) *TestConnector { + return &TestConnector{&connector{ + log: zap.NewNop(), + cfg: cfg, + client: client, + tts: tts, + }} +} + +func (c *TestConnector) Connect(ctx context.Context, in map[string]any) (any, error) { + return c.connector.Connect(ctx, in) +} diff --git a/plugins/actions/luma_checkin/luma_checkin.go b/plugins/actions/luma_checkin/luma_checkin.go new file mode 100644 index 0000000000..ce8736cac1 --- /dev/null +++ b/plugins/actions/luma_checkin/luma_checkin.go @@ -0,0 +1,291 @@ +package luma_checkin + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/openmind/om1/internal/actions" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers/tts" +) + +const ( + defaultGreetingTemplate = "Welcome, {first_name}! Glad you made it." + defaultStatus = "checked_in" + sideEffectTimeout = 5 * time.Second +) + +// LumaCheckinInput is the LLM-visible argument schema. The event is configured +// per-deployment; the LLM only supplies the pk parsed from the QR code. +type LumaCheckinInput struct { + Pk string `json:"pk" description:"The Luma guest or ticket key (pk parameter from check-in QR code)"` +} + +// ttsConfig configures which TTS provider to use for the welcome utterance. +// Both Kokoro and ElevenLabs providers are sync.Once singletons, so this +// instance is shared with the speak action plugin if both are configured; +// the first caller's config wins. +type ttsConfig struct { + Provider string `json:"provider"` + APIKey string `json:"api_key"` + ElevenLabsAPIKey string `json:"elevenlabs_api_key"` + BaseURL string `json:"base_url"` + VoiceID string `json:"voice_id"` + ModelID string `json:"model_id"` + OutputFormat string `json:"output_format"` + Rate int `json:"rate"` +} + +// Config holds the plugin configuration decoded from JSON5. +type Config struct { + APIKey string `json:"api_key"` + BaseURL string `json:"base_url"` + EventAPIID string `json:"event_api_id"` + GreetingTemplate string `json:"greeting_template"` + RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` + StatusValue string `json:"status_value"` + TTS ttsConfig `json:"tts"` +} + +// ttsPlayer is the minimal interface luma_checkin needs from a TTS provider. +type ttsPlayer interface { + AddText(text string) +} + +// guestLookup is the interface luma_checkin uses against Luma; abstracted for tests. +type guestLookup interface { + GetGuest(ctx context.Context, pk string) (*Guest, error) + UpdateGuestStatus(ctx context.Context, eventAPIID, guestAPIID, status string) error +} + +type connector struct { + log *zap.Logger + cfg Config + client guestLookup + tts ttsPlayer + + mu sync.Mutex + stopped bool +} + +func init() { + actions.RegisterInterface( + "luma_checkin", + "Action interface for checking guests into a Luma event. "+ + "Looks up the guest via the Luma external check-in API, validates the configured event, "+ + "flips status to checked_in, and speaks a personalized welcome via the configured TTS provider. "+ + "For arm gestures and facial expressions, call unitree_g1_arm and emotion alongside this action.", + LumaCheckinInput{}, + ) + actions.Register("luma_checkin/api", NewLumaCheckinConnector) +} + +// NewLumaCheckinConnector constructs a connector from the decoded config map. +func NewLumaCheckinConnector(cfgMap map[string]any) (actions.Connector, error) { + cfg, err := parseConfig(cfgMap) + if err != nil { + return nil, err + } + log := logger.Get().Named("luma_checkin/api") + + timeout := time.Duration(cfg.RequestTimeoutSeconds * float64(time.Second)) + c := &connector{ + log: log, + cfg: cfg, + client: newLumaClient(cfg.BaseURL, cfg.APIKey, cfg.EventAPIID, timeout), + tts: buildTTS(cfg.TTS, log), + } + + log.Info("initialized", + zap.String("event_api_id", cfg.EventAPIID), + zap.String("tts_provider", cfg.TTS.Provider), + ) + return c, nil +} + +func parseConfig(cfgMap map[string]any) (Config, error) { + var cfg Config + if b, err := json.Marshal(cfgMap); err == nil { + _ = json.Unmarshal(b, &cfg) + } + if cfg.APIKey == "" { + return cfg, fmt.Errorf("luma_checkin: api_key is required") + } + if cfg.EventAPIID == "" { + return cfg, fmt.Errorf("luma_checkin: event_api_id is required") + } + if cfg.GreetingTemplate == "" { + cfg.GreetingTemplate = defaultGreetingTemplate + } + if cfg.StatusValue == "" { + cfg.StatusValue = defaultStatus + } + if cfg.RequestTimeoutSeconds <= 0 { + cfg.RequestTimeoutSeconds = 5 + } + return cfg, nil +} + +func buildTTS(cfg ttsConfig, log *zap.Logger) ttsPlayer { + switch strings.ToLower(cfg.Provider) { + case "kokoro": + return tts.Kokoro(tts.KokoroConfig{ + BaseURL: orDefault(cfg.BaseURL, tts.DefaultKokoroBaseURL), + APIKey: cfg.APIKey, + VoiceID: orDefault(cfg.VoiceID, tts.DefaultKokoroVoiceID), + ModelID: orDefault(cfg.ModelID, tts.DefaultKokoroModelID), + OutputFormat: orDefault(cfg.OutputFormat, tts.DefaultKokoroOutputFormat), + Rate: orDefaultInt(cfg.Rate, tts.DefaultKokoroRate), + }, log) + case "elevenlabs": + return tts.ElevenLabs(tts.ElevenLabsConfig{ + APIKey: cfg.APIKey, + ElevenLabsAPIKey: cfg.ElevenLabsAPIKey, + VoiceID: orDefault(cfg.VoiceID, tts.DefaultVoiceID), + ModelID: orDefault(cfg.ModelID, tts.DefaultModelID), + OutputFormat: orDefault(cfg.OutputFormat, tts.DefaultOutputFormat), + Rate: orDefaultInt(cfg.Rate, tts.DefaultRate), + }, log) + case "": + return nil + default: + log.Warn("luma_checkin: unknown tts.provider", zap.String("provider", cfg.Provider)) + return nil + } +} + +// Connect performs the check-in: GetGuest → validate event → fan out the +// status update + TTS welcome → return status string. Arm wave and emotion +// are intentionally not handled here; configure unitree_g1_arm and emotion +// actions and let the LLM call them concurrently. +func (c *connector) Connect(ctx context.Context, input actions.Input) (actions.Output, error) { + args, ok := input.(map[string]any) + if !ok { + return nil, fmt.Errorf("luma_checkin: unexpected input type %T", input) + } + pk := strings.TrimSpace(stringField(args, "pk")) + if pk == "" { + return nil, fmt.Errorf("luma_checkin: pk required") + } + + guest, err := c.client.GetGuest(ctx, pk) + if err != nil { + switch { + case errors.Is(err, errLumaNotFound): + c.log.Info("guest not found", zap.String("pk", pk)) + return "checkin_failed: guest not registered for this event", nil + case errors.Is(err, errLumaUnauthorized): + c.log.Error("luma auth failed") + return nil, fmt.Errorf("luma api auth failed") + default: + c.log.Warn("get-guest failed", zap.Error(err)) + return nil, err + } + } + if guest == nil { + return "checkin_failed: empty guest response", nil + } + if guest.EventAPIID != "" && guest.EventAPIID != c.cfg.EventAPIID { + c.log.Info("event mismatch", + zap.String("expected", c.cfg.EventAPIID), + zap.String("got", guest.EventAPIID), + ) + return "checkin_failed: qr is for a different event", nil + } + + displayName := firstNameFor(guest) + + sideCtx, cancel := context.WithTimeout(ctx, sideEffectTimeout) + defer cancel() + + var wg sync.WaitGroup + var statusErr error + var statusMu sync.Mutex + + if guest.APIID != "" { + wg.Add(1) + go func() { + defer wg.Done() + if err := c.client.UpdateGuestStatus(sideCtx, c.cfg.EventAPIID, guest.APIID, c.cfg.StatusValue); err != nil { + c.log.Warn("update-guest-status failed", zap.Error(err)) + statusMu.Lock() + statusErr = err + statusMu.Unlock() + } + }() + } + + if c.tts != nil { + c.tts.AddText(formatGreeting(c.cfg.GreetingTemplate, guest)) + } + + wg.Wait() + + statusMu.Lock() + failed := statusErr != nil + statusMu.Unlock() + + c.log.Info("checked in", zap.String("pk", pk), zap.String("name", displayName), zap.Bool("status_update_failed", failed)) + if failed { + return fmt.Sprintf("checked_in: %s (status update failed, logged)", displayName), nil + } + return fmt.Sprintf("checked_in: %s", displayName), nil +} + +func (c *connector) Tick(ctx context.Context) { + <-ctx.Done() +} + +func (c *connector) Stop() { + c.mu.Lock() + defer c.mu.Unlock() + c.stopped = true +} + +func stringField(m map[string]any, key string) string { + v, _ := m[key].(string) + return v +} + +func firstNameFor(g *Guest) string { + if g.FirstName != "" { + return g.FirstName + } + if g.Name != "" { + if parts := strings.Fields(g.Name); len(parts) > 0 { + return parts[0] + } + } + return "friend" +} + +func formatGreeting(template string, g *Guest) string { + r := strings.NewReplacer( + "{first_name}", firstNameFor(g), + "{last_name}", g.LastName, + "{name}", orDefault(g.Name, firstNameFor(g)), + "{email}", g.Email, + ) + return r.Replace(template) +} + +func orDefault(v, def string) string { + if v != "" { + return v + } + return def +} + +func orDefaultInt(v, def int) int { + if v > 0 { + return v + } + return def +} diff --git a/plugins/actions/luma_checkin/luma_checkin_test.go b/plugins/actions/luma_checkin/luma_checkin_test.go new file mode 100644 index 0000000000..20a1f711d8 --- /dev/null +++ b/plugins/actions/luma_checkin/luma_checkin_test.go @@ -0,0 +1,196 @@ +package luma_checkin_test + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/openmind/om1/plugins/actions/luma_checkin" +) + +type fakeGuestClient struct { + guest *luma_checkin.Guest + getErr error + updateErr error + getCalls int32 + updateCalls int32 + lastStatusArgs struct { + eventAPIID, guestAPIID, status string + } + mu sync.Mutex +} + +func (f *fakeGuestClient) GetGuest(_ context.Context, _ string) (*luma_checkin.Guest, error) { + atomic.AddInt32(&f.getCalls, 1) + return f.guest, f.getErr +} + +func (f *fakeGuestClient) UpdateGuestStatus(_ context.Context, eventAPIID, guestAPIID, status string) error { + atomic.AddInt32(&f.updateCalls, 1) + f.mu.Lock() + f.lastStatusArgs.eventAPIID = eventAPIID + f.lastStatusArgs.guestAPIID = guestAPIID + f.lastStatusArgs.status = status + f.mu.Unlock() + return f.updateErr +} + +type fakeTTS struct { + mu sync.Mutex + texts []string +} + +func (f *fakeTTS) AddText(t string) { + f.mu.Lock() + defer f.mu.Unlock() + f.texts = append(f.texts, t) +} + +func (f *fakeTTS) lastText() string { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.texts) == 0 { + return "" + } + return f.texts[len(f.texts)-1] +} + +func newTestConnector(client luma_checkin.GuestLookup, cfg luma_checkin.Config) (*luma_checkin.TestConnector, *fakeTTS) { + tts := &fakeTTS{} + c := luma_checkin.NewTestConnector(client, cfg, tts) + return c, tts +} + +func defaultCfg() luma_checkin.Config { + return luma_checkin.Config{ + APIKey: "k", + EventAPIID: "evt-1", + GreetingTemplate: luma_checkin.DefaultGreetingTemplate, + StatusValue: luma_checkin.DefaultStatus, + } +} + +func TestConnectHappyPath(t *testing.T) { + client := &fakeGuestClient{guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-1"}} + c, tts := newTestConnector(client, defaultCfg()) + + out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) + if err != nil { + t.Fatalf("Connect: %v", err) + } + if got, want := out.(string), "checked_in: Ada"; got != want { + t.Errorf("output: got %q want %q", got, want) + } + if atomic.LoadInt32(&client.updateCalls) != 1 { + t.Errorf("expected 1 update call, got %d", client.updateCalls) + } + if client.lastStatusArgs.guestAPIID != "g-1" || client.lastStatusArgs.status != luma_checkin.DefaultStatus { + t.Errorf("update args: %+v", client.lastStatusArgs) + } + if !strings.Contains(tts.lastText(), "Ada") { + t.Errorf("tts text missing first name: %q", tts.lastText()) + } +} + +func TestConnectEventMismatch(t *testing.T) { + client := &fakeGuestClient{guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-other"}} + c, tts := newTestConnector(client, defaultCfg()) + + out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) + if err != nil { + t.Fatalf("Connect: %v", err) + } + if !strings.HasPrefix(out.(string), "checkin_failed:") { + t.Errorf("output: %q", out) + } + if atomic.LoadInt32(&client.updateCalls) != 0 { + t.Errorf("update should not be called on mismatch") + } + if tts.lastText() != "" { + t.Errorf("tts should not fire on mismatch") + } +} + +func TestConnectStatusUpdateFailureNonBlocking(t *testing.T) { + client := &fakeGuestClient{ + guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-1"}, + updateErr: errors.New("boom"), + } + c, tts := newTestConnector(client, defaultCfg()) + + out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) + if err != nil { + t.Fatalf("Connect: %v", err) + } + s := out.(string) + if !strings.HasPrefix(s, "checked_in: Ada") || !strings.Contains(s, "status update failed") { + t.Errorf("output: %q", s) + } + if !strings.Contains(tts.lastText(), "Ada") { + t.Errorf("tts should still fire when status update fails: %q", tts.lastText()) + } +} + +func TestConnectGuestNotFound(t *testing.T) { + client := &fakeGuestClient{getErr: luma_checkin.ErrLumaNotFound} + c, _ := newTestConnector(client, defaultCfg()) + + out, err := c.Connect(context.Background(), map[string]any{"pk": "g-x"}) + if err != nil { + t.Fatalf("Connect: %v", err) + } + if !strings.HasPrefix(out.(string), "checkin_failed:") { + t.Errorf("output: %q", out) + } +} + +func TestConnectAuthFailure(t *testing.T) { + client := &fakeGuestClient{getErr: luma_checkin.ErrLumaUnauthorized} + c, _ := newTestConnector(client, defaultCfg()) + + _, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) + if err == nil { + t.Fatalf("expected error on auth failure") + } +} + +func TestConnectEmptyPk(t *testing.T) { + client := &fakeGuestClient{} + c, _ := newTestConnector(client, defaultCfg()) + + _, err := c.Connect(context.Background(), map[string]any{"pk": ""}) + if err == nil { + t.Fatalf("expected error on empty pk") + } + if atomic.LoadInt32(&client.getCalls) != 0 { + t.Errorf("should not call GetGuest with empty pk") + } +} + +func TestFirstNameFallbacks(t *testing.T) { + cases := []struct { + guest *luma_checkin.Guest + want string + }{ + {&luma_checkin.Guest{FirstName: "Ada"}, "Ada"}, + {&luma_checkin.Guest{Name: "Ada Lovelace"}, "Ada"}, + {&luma_checkin.Guest{}, "friend"}, + } + for _, tc := range cases { + if got := luma_checkin.FirstNameFor(tc.guest); got != tc.want { + t.Errorf("FirstNameFor(%+v): got %q want %q", tc.guest, got, tc.want) + } + } +} + +func TestFormatGreetingTokens(t *testing.T) { + g := &luma_checkin.Guest{FirstName: "Ada", LastName: "Lovelace", Name: "Ada Lovelace", Email: "ada@example.com"} + got := luma_checkin.FormatGreeting("Hi {first_name} {last_name} <{email}>", g) + want := "Hi Ada Lovelace " + if got != want { + t.Errorf("got %q want %q", got, want) + } +} diff --git a/plugins/actions/luma_checkin/luma_client.go b/plugins/actions/luma_checkin/luma_client.go new file mode 100644 index 0000000000..8eacdacc35 --- /dev/null +++ b/plugins/actions/luma_checkin/luma_client.go @@ -0,0 +1,198 @@ +package luma_checkin + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/openmind/om1/internal/httpclient" +) + +const ( + defaultLumaBaseURL = "https://public-api.luma.com" + getGuestPath = "/v1/event/get-guest" + updateGuestStatusPath = "/v1/event/update-guest-status" +) + +var ( + errLumaNotFound = errors.New("luma: guest not found") + errLumaUnauthorized = errors.New("luma: unauthorized") +) + +// Guest is a flattened view of the guest record returned by Luma. Field names +// follow the API's snake_case JSON. Unknown fields are ignored. +type Guest struct { + APIID string `json:"api_id"` + Name string `json:"name"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Email string `json:"email"` + EventAPIID string `json:"event_api_id"` + CheckedInAt string `json:"checked_in_at"` + ApprovalStatus string `json:"approval_status"` +} + +// guestEnvelope handles Luma's nested response shape: {"guest": {...}, "event": {...}}. +// Some endpoints return the guest object directly; we try the envelope first. +type guestEnvelope struct { + Guest *Guest `json:"guest"` + Event *struct { + APIID string `json:"api_id"` + } `json:"event"` +} + +// httpDoer is satisfied by *http.Client; abstracted for tests. +type httpDoer interface { + Do(req *http.Request) (*http.Response, error) +} + +type lumaClient struct { + baseURL string + apiKey string + eventAPIID string + http httpDoer + timeout time.Duration +} + +func newLumaClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *lumaClient { + if baseURL == "" { + baseURL = defaultLumaBaseURL + } + if timeout <= 0 { + timeout = 5 * time.Second + } + return &lumaClient{ + baseURL: strings.TrimRight(baseURL, "/"), + apiKey: apiKey, + eventAPIID: eventAPIID, + http: httpclient.Default(), + timeout: timeout, + } +} + +// GetGuest fetches the guest record for the given key (g- or ticket key). +// Luma's get-guest requires both event_id (the configured event) and id (the +// guest/ticket key from the QR). +func (c *lumaClient) GetGuest(ctx context.Context, pk string) (*Guest, error) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + q := url.Values{ + "event_id": []string{c.eventAPIID}, + "id": []string{pk}, + } + endpoint := c.baseURL + getGuestPath + "?" + q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + c.setAuth(req) + + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + switch resp.StatusCode { + case http.StatusOK: + // fallthrough to decode + case http.StatusNotFound: + return nil, errLumaNotFound + case http.StatusUnauthorized, http.StatusForbidden: + return nil, errLumaUnauthorized + default: + return nil, fmt.Errorf("luma get-guest %d: %s", resp.StatusCode, string(body)) + } + + var env guestEnvelope + if err := json.Unmarshal(body, &env); err == nil && env.Guest != nil { + g := env.Guest + if g.EventAPIID == "" && env.Event != nil { + g.EventAPIID = env.Event.APIID + } + return g, nil + } + var bare Guest + if err := json.Unmarshal(body, &bare); err != nil { + return nil, fmt.Errorf("luma get-guest decode: %w", err) + } + return &bare, nil +} + +// UpdateGuestStatus flips the guest's approval/check-in status. Best-effort: the +// exact body shape Luma expects is under-documented, so the action treats +// failures here as non-fatal. +func (c *lumaClient) UpdateGuestStatus(ctx context.Context, eventAPIID, guestAPIID, status string) error { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + body, _ := json.Marshal(map[string]any{ + "event_id": eventAPIID, + "id": guestAPIID, + "status": status, + }) + endpoint := c.baseURL + updateGuestStatusPath + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + c.setAuth(req) + + resp, err := c.do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + rb, _ := io.ReadAll(resp.Body) + return fmt.Errorf("luma update-guest-status %d: %s", resp.StatusCode, string(rb)) +} + +func (c *lumaClient) setAuth(req *http.Request) { + req.Header.Set("x-luma-api-key", c.apiKey) + req.Header.Set("Accept", "application/json") +} + +// do issues req with one retry on 429 after a short backoff. +func (c *lumaClient) do(req *http.Request) (*http.Response, error) { + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusTooManyRequests { + return resp, nil + } + + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + t := time.NewTimer(500 * time.Millisecond) + defer t.Stop() + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-t.C: + } + + if req.GetBody != nil { + body, berr := req.GetBody() + if berr != nil { + return nil, berr + } + req.Body = body + } + return c.http.Do(req) +} diff --git a/plugins/actions/luma_checkin/luma_client_test.go b/plugins/actions/luma_checkin/luma_client_test.go new file mode 100644 index 0000000000..d828a50ea8 --- /dev/null +++ b/plugins/actions/luma_checkin/luma_client_test.go @@ -0,0 +1,155 @@ +package luma_checkin_test + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/openmind/om1/plugins/actions/luma_checkin" +) + +func newTestClient(t *testing.T, handler http.HandlerFunc) (*luma_checkin.LumaClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + c := luma_checkin.NewLumaClient(srv.URL, "test-key", "evt-test", 2*time.Second) + return c, srv +} + +func TestGetGuestEnvelopeOK(t *testing.T) { + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != luma_checkin.GetGuestPath { + t.Errorf("path: got %q want %q", r.URL.Path, luma_checkin.GetGuestPath) + } + if r.URL.Query().Get("id") != "g-123" { + t.Errorf("id query: got %q want g-123", r.URL.Query().Get("id")) + } + if r.URL.Query().Get("event_id") != "evt-test" { + t.Errorf("event_id query: got %q want evt-test", r.URL.Query().Get("event_id")) + } + if r.Header.Get("x-luma-api-key") != "test-key" { + t.Errorf("missing api key header: %q", r.Header.Get("x-luma-api-key")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"guest":{"api_id":"g-123","first_name":"Ada","name":"Ada Lovelace","email":"ada@example.com"},"event":{"api_id":"evt-abc"}}`) + }) + defer srv.Close() + + g, err := c.GetGuest(context.Background(), "g-123") + if err != nil { + t.Fatalf("GetGuest: %v", err) + } + if g.APIID != "g-123" || g.FirstName != "Ada" || g.EventAPIID != "evt-abc" { + t.Errorf("unexpected guest: %+v", g) + } +} + +func TestGetGuestBareOK(t *testing.T) { + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{"api_id":"g-9","first_name":"Bo","event_api_id":"evt-1"}`) + }) + defer srv.Close() + + g, err := c.GetGuest(context.Background(), "g-9") + if err != nil { + t.Fatalf("GetGuest: %v", err) + } + if g.FirstName != "Bo" || g.EventAPIID != "evt-1" { + t.Errorf("unexpected guest: %+v", g) + } +} + +func TestGetGuestNotFound(t *testing.T) { + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + defer srv.Close() + + _, err := c.GetGuest(context.Background(), "g-x") + if !errors.Is(err, luma_checkin.ErrLumaNotFound) { + t.Fatalf("want ErrLumaNotFound, got %v", err) + } +} + +func TestGetGuestUnauthorized(t *testing.T) { + for _, code := range []int{http.StatusUnauthorized, http.StatusForbidden} { + code := code + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + }) + _, err := c.GetGuest(context.Background(), "g-x") + srv.Close() + if !errors.Is(err, luma_checkin.ErrLumaUnauthorized) { + t.Errorf("status %d: want ErrLumaUnauthorized, got %v", code, err) + } + } +} + +func TestGetGuestRetriesOn429(t *testing.T) { + var calls int + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = io.WriteString(w, `{"api_id":"g-1","first_name":"A","event_api_id":"e"}`) + }) + defer srv.Close() + + g, err := c.GetGuest(context.Background(), "g-1") + if err != nil { + t.Fatalf("GetGuest: %v", err) + } + if calls != 2 { + t.Errorf("expected 2 attempts, got %d", calls) + } + if g.FirstName != "A" { + t.Errorf("unexpected guest: %+v", g) + } +} + +func TestUpdateGuestStatusOK(t *testing.T) { + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method: got %s want POST", r.Method) + } + if r.URL.Path != luma_checkin.UpdateGuestStatusPath { + t.Errorf("path: got %q want %q", r.URL.Path, luma_checkin.UpdateGuestStatusPath) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Errorf("content-type: %q", r.Header.Get("Content-Type")) + } + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body["event_id"] != "evt-1" || body["id"] != "g-1" || body["status"] != "checked_in" { + t.Errorf("body: %+v", body) + } + w.WriteHeader(http.StatusOK) + }) + defer srv.Close() + + if err := c.UpdateGuestStatus(context.Background(), "evt-1", "g-1", "checked_in"); err != nil { + t.Fatalf("UpdateGuestStatus: %v", err) + } +} + +func TestUpdateGuestStatusErrorBubbles(t *testing.T) { + c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"error":"bad shape"}`) + }) + defer srv.Close() + + err := c.UpdateGuestStatus(context.Background(), "evt-1", "g-1", "checked_in") + if err == nil || !strings.Contains(err.Error(), "400") { + t.Fatalf("want 400 error, got %v", err) + } +} diff --git a/plugins/inputs/inputs.go b/plugins/inputs/inputs.go index b612cbf00b..818000c1cd 100644 --- a/plugins/inputs/inputs.go +++ b/plugins/inputs/inputs.go @@ -2,6 +2,7 @@ package inputs import ( _ "github.com/openmind/om1/plugins/inputs/asr" + _ "github.com/openmind/om1/plugins/inputs/qr_scanner" _ "github.com/openmind/om1/plugins/inputs/unitree/go2" _ "github.com/openmind/om1/plugins/inputs/vlm" ) diff --git a/plugins/inputs/qr_scanner/debounce.go b/plugins/inputs/qr_scanner/debounce.go new file mode 100644 index 0000000000..84901c1075 --- /dev/null +++ b/plugins/inputs/qr_scanner/debounce.go @@ -0,0 +1,47 @@ +package qr_scanner + +import ( + "sync" + "time" +) + +// debouncer drops repeated `pk` values seen within a sliding time window. +type debouncer struct { + mu sync.Mutex + window time.Duration + seen map[string]time.Time + now func() time.Time +} + +func newDebouncer(window time.Duration) *debouncer { + return &debouncer{ + window: window, + seen: make(map[string]time.Time), + now: time.Now, + } +} + +// TryRecord returns true if pk was not seen within the configured window. +// Recording also prunes entries older than 10x the window to bound memory. +func (d *debouncer) TryRecord(pk string) bool { + d.mu.Lock() + defer d.mu.Unlock() + + now := d.now() + d.pruneLocked(now) + + if last, ok := d.seen[pk]; ok && now.Sub(last) < d.window { + return false + } + d.seen[pk] = now + return true +} + +func (d *debouncer) pruneLocked(now time.Time) { + cutoff := now.Add(-10 * d.window) + for k, t := range d.seen { + if t.Before(cutoff) { + delete(d.seen, k) + } + } +} diff --git a/plugins/inputs/qr_scanner/debounce_test.go b/plugins/inputs/qr_scanner/debounce_test.go new file mode 100644 index 0000000000..52e424bf5d --- /dev/null +++ b/plugins/inputs/qr_scanner/debounce_test.go @@ -0,0 +1,62 @@ +package qr_scanner_test + +import ( + "testing" + "time" + + "github.com/openmind/om1/plugins/inputs/qr_scanner" +) + +func TestDebouncerAcceptsThenRejects(t *testing.T) { + d := qr_scanner.NewDebouncer(30 * time.Second) + now := time.Unix(1_700_000_000, 0) + d.SetNow(func() time.Time { return now }) + + if !d.TryRecord("g-1") { + t.Fatalf("first record should be accepted") + } + if d.TryRecord("g-1") { + t.Fatalf("immediate re-record should be rejected") + } + + now = now.Add(29 * time.Second) + if d.TryRecord("g-1") { + t.Fatalf("re-record within window should be rejected") + } + + now = now.Add(2 * time.Second) // total 31s after first + if !d.TryRecord("g-1") { + t.Fatalf("re-record after window should be accepted") + } +} + +func TestDebouncerDistinctKeysIndependent(t *testing.T) { + d := qr_scanner.NewDebouncer(30 * time.Second) + if !d.TryRecord("g-1") { + t.Fatalf("g-1 first should be accepted") + } + if !d.TryRecord("g-2") { + t.Fatalf("g-2 first should be accepted") + } +} + +func TestDebouncerPrunesOldEntries(t *testing.T) { + d := qr_scanner.NewDebouncer(1 * time.Second) + now := time.Unix(1_700_000_000, 0) + d.SetNow(func() time.Time { return now }) + + d.TryRecord("g-old") + if !d.Has("g-old") { + t.Fatalf("expected g-old to be recorded") + } + + now = now.Add(15 * time.Second) // 15x window + d.TryRecord("g-fresh") + + if d.Has("g-old") { + t.Fatalf("expected g-old to be pruned after 15x window") + } + if !d.Has("g-fresh") { + t.Fatalf("expected g-fresh to be present") + } +} diff --git a/plugins/inputs/qr_scanner/decode.go b/plugins/inputs/qr_scanner/decode.go new file mode 100644 index 0000000000..446e8d81e7 --- /dev/null +++ b/plugins/inputs/qr_scanner/decode.go @@ -0,0 +1,34 @@ +package qr_scanner + +import ( + "bytes" + "errors" + "image" + _ "image/jpeg" + + "github.com/makiuchi-d/gozxing" + "github.com/makiuchi-d/gozxing/qrcode" +) + +// errQRNotFound signals that no QR code was found in the frame. Callers should +// treat this as the common case and skip silently. +var errQRNotFound = errors.New("qr_scanner: no qr code in frame") + +// decodeQR decodes a single QR code from a JPEG-encoded frame. It returns the +// raw text payload of the code, or errQRNotFound if no code is present. +func decodeQR(jpegBytes []byte) (string, error) { + img, _, err := image.Decode(bytes.NewReader(jpegBytes)) + if err != nil { + return "", err + } + bmp, err := gozxing.NewBinaryBitmapFromImage(img) + if err != nil { + return "", err + } + reader := qrcode.NewQRCodeReader() + result, err := reader.Decode(bmp, nil) + if err != nil { + return "", errQRNotFound + } + return result.GetText(), nil +} diff --git a/plugins/inputs/qr_scanner/decode_test.go b/plugins/inputs/qr_scanner/decode_test.go new file mode 100644 index 0000000000..d1f937c213 --- /dev/null +++ b/plugins/inputs/qr_scanner/decode_test.go @@ -0,0 +1,73 @@ +package qr_scanner_test + +import ( + "bytes" + "errors" + "image" + "image/color" + "image/jpeg" + "testing" + + "github.com/makiuchi-d/gozxing" + "github.com/makiuchi-d/gozxing/qrcode" + + "github.com/openmind/om1/plugins/inputs/qr_scanner" +) + +func TestDecodeQRRoundTrip(t *testing.T) { + const payload = "https://luma.com/check-in/evt-test?pk=g-test-123" + + jpegBytes := encodeQRAsJPEG(t, payload, 360) + + got, err := qr_scanner.DecodeQR(jpegBytes) + if err != nil { + t.Fatalf("DecodeQR: %v", err) + } + if got != payload { + t.Errorf("payload: got %q want %q", got, payload) + } +} + +func TestDecodeQRReturnsNotFoundOnBlankFrame(t *testing.T) { + jpegBytes := encodeBlankJPEG(t, 100, 100) + + _, err := qr_scanner.DecodeQR(jpegBytes) + if !errors.Is(err, qr_scanner.ErrQRNotFound) { + t.Fatalf("expected ErrQRNotFound, got %v", err) + } +} + +func encodeQRAsJPEG(t *testing.T, payload string, size int) []byte { + t.Helper() + writer := qrcode.NewQRCodeWriter() + bm, err := writer.Encode(payload, gozxing.BarcodeFormat_QR_CODE, size, size, nil) + if err != nil { + t.Fatalf("encode QR: %v", err) + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, bm, &jpeg.Options{Quality: 90}); err != nil { + t.Fatalf("jpeg encode: %v", err) + } + return buf.Bytes() +} + +func encodeBlankJPEG(t *testing.T, w, h int) []byte { + t.Helper() + img := newBlankImage(w, h) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + t.Fatalf("jpeg encode: %v", err) + } + return buf.Bytes() +} + +func newBlankImage(w, h int) image.Image { + img := image.NewGray(image.Rect(0, 0, w, h)) + white := color.Gray{Y: 255} + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.SetGray(x, y, white) + } + } + return img +} diff --git a/plugins/inputs/qr_scanner/export_test.go b/plugins/inputs/qr_scanner/export_test.go new file mode 100644 index 0000000000..c3f69458aa --- /dev/null +++ b/plugins/inputs/qr_scanner/export_test.go @@ -0,0 +1,16 @@ +package qr_scanner + +import "time" + +var ( + ParseLumaCheckinURL = parseLumaCheckinURL + DecodeQR = decodeQR + ErrQRNotFound = errQRNotFound +) + +type Debouncer = debouncer + +func NewDebouncer(window time.Duration) *Debouncer { return newDebouncer(window) } + +func (d *Debouncer) SetNow(f func() time.Time) { d.now = f } +func (d *Debouncer) Has(key string) bool { _, ok := d.seen[key]; return ok } diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go new file mode 100644 index 0000000000..0e6caef4e1 --- /dev/null +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -0,0 +1,243 @@ +package qr_scanner + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/openmind/om1/internal/inputs" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers" + video "github.com/openmind/om1/internal/providers/vlm" +) + +const ( + scannerName = "QRScanner" + scannerDescriptor = "QR Scanner" + scannerMaxMessages = 8 + scanChannelBuffer = 4 +) + +func init() { + inputs.Register(scannerName, NewQRScanner) +} + +// Config holds the JSON configuration for the QRScanner input plugin. +type Config struct { + CameraIndex int `json:"camera_index"` + CaptureFPS int `json:"capture_fps"` + DecodeFPS int `json:"decode_fps"` + Width int `json:"resolution_width"` + Height int `json:"resolution_height"` + JPEGQuality int `json:"jpeg_quality"` + DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` +} + +type frameSource interface { + Start(ctx context.Context) <-chan video.Frame + Stop() +} + +type sensor struct { + cfg Config + log *zap.Logger + source frameSource + debouncer *debouncer + + mu sync.Mutex + messages []inputs.Message + stopped bool + cancel context.CancelFunc +} + +// NewQRScanner constructs a QRScanner sensor from the decoded config map. +func NewQRScanner(configMap map[string]any) (inputs.Sensor, error) { + cfg := parseConfig(configMap) + log := logger.Get().Named(scannerName) + log.Info("initializing", + zap.Int("camera_index", cfg.CameraIndex), + zap.Int("capture_fps", cfg.CaptureFPS), + zap.Int("decode_fps", cfg.DecodeFPS), + zap.Int("width", cfg.Width), + zap.Int("height", cfg.Height), + zap.Float64("dedupe_window_seconds", cfg.DedupeWindowSeconds), + ) + + source := video.NewVideoStream(video.VideoStreamConfig{ + DeviceIndex: cfg.CameraIndex, + FPS: cfg.CaptureFPS, + Width: cfg.Width, + Height: cfg.Height, + JPEGQuality: cfg.JPEGQuality, + }) + + window := time.Duration(cfg.DedupeWindowSeconds * float64(time.Second)) + return &sensor{ + cfg: cfg, + log: log, + source: source, + debouncer: newDebouncer(window), + }, nil +} + +func parseConfig(configMap map[string]any) Config { + var cfg Config + if b, err := json.Marshal(configMap); err == nil { + _ = json.Unmarshal(b, &cfg) + } + if cfg.CaptureFPS <= 0 { + cfg.CaptureFPS = 15 + } + if cfg.DecodeFPS <= 0 { + cfg.DecodeFPS = 5 + } + if cfg.DecodeFPS > cfg.CaptureFPS { + cfg.DecodeFPS = cfg.CaptureFPS + } + if cfg.Width <= 0 { + cfg.Width = 640 + } + if cfg.Height <= 0 { + cfg.Height = 480 + } + if cfg.JPEGQuality <= 0 { + cfg.JPEGQuality = 60 + } + if cfg.DedupeWindowSeconds <= 0 { + cfg.DedupeWindowSeconds = 30 + } + return cfg +} + +// Listen starts the camera and emits scanned `pk` values as text on the returned channel. +func (s *sensor) Listen(ctx context.Context) (<-chan any, error) { + ctx, cancel := context.WithCancel(ctx) + s.mu.Lock() + s.cancel = cancel + s.mu.Unlock() + + frames := s.source.Start(ctx) + out := make(chan any, scanChannelBuffer) + + stride := s.cfg.CaptureFPS / s.cfg.DecodeFPS + if stride < 1 { + stride = 1 + } + + go func() { + defer close(out) + defer s.Stop() + + var counter uint64 + for { + select { + case <-ctx.Done(): + return + case frame, ok := <-frames: + if !ok { + return + } + counter++ + if counter%uint64(stride) != 0 { + continue + } + + text, err := decodeQR(frame.JPEG) + if err != nil { + continue + } + eventID, pk, valid := parseLumaCheckinURL(text) + if !valid { + s.log.Debug("ignoring non-luma qr", zap.String("text", truncate(text, 80))) + continue + } + if !s.debouncer.TryRecord(pk) { + s.log.Debug("debounced", zap.String("pk", pk)) + continue + } + + msg := fmt.Sprintf("qr_scan: pk=%s event=%s", pk, eventID) + s.log.Info("emitted scan", zap.String("pk", pk), zap.String("event", eventID)) + select { + case out <- msg: + default: + s.log.Warn("scan channel full, dropping", zap.String("pk", pk)) + } + } + } + }() + + return out, nil +} + +func (s *sensor) Poll(context.Context) (any, error) { return nil, nil } + +// RawToText converts a raw scan event into a timestamped Message and appends it +// to the bounded in-memory history. +func (s *sensor) RawToText(_ context.Context, raw any) (*inputs.Message, error) { + text, ok := raw.(string) + if !ok || text == "" { + return nil, nil + } + msg := inputs.NewMessage(text) + + s.mu.Lock() + s.messages = append(s.messages, *msg) + if len(s.messages) > scannerMaxMessages { + s.messages = s.messages[len(s.messages)-scannerMaxMessages:] + } + s.mu.Unlock() + + return msg, nil +} + +// FormattedLatestBuffer returns the newest scan formatted for the LLM prompt +// and clears the history. Returns "" when empty. +func (s *sensor) FormattedLatestBuffer() string { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.messages) == 0 { + return "" + } + + latest := s.messages[len(s.messages)-1] + result := fmt.Sprintf("\n%s: '%s'\n", scannerDescriptor, latest.Message) + + ts := time.Unix(0, int64(latest.Timestamp*1e9)) + providers.IO().AddInput(scannerName, latest.Message, ts) + s.messages = nil + + return result +} + +// TriggersTick opts the scanner into waking the cortex loop on every fresh scan. +func (s *sensor) TriggersTick() bool { return true } + +func (s *sensor) Stop() { + s.mu.Lock() + if s.stopped { + s.mu.Unlock() + return + } + s.stopped = true + cancel := s.cancel + s.mu.Unlock() + + if cancel != nil { + cancel() + } + s.source.Stop() + s.log.Info("stopping sensor") +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/plugins/inputs/qr_scanner/url_parse.go b/plugins/inputs/qr_scanner/url_parse.go new file mode 100644 index 0000000000..c5aabfa68e --- /dev/null +++ b/plugins/inputs/qr_scanner/url_parse.go @@ -0,0 +1,32 @@ +package qr_scanner + +import ( + "net/url" + "strings" +) + +// parseLumaCheckinURL extracts the event ID and pk from a Luma check-in URL of +// the form https://luma.com/check-in/?pk=. Accepted hosts are +// luma.com, www.luma.com, and lu.ma. +func parseLumaCheckinURL(s string) (eventID, pk string, ok bool) { + u, err := url.Parse(strings.TrimSpace(s)) + if err != nil { + return "", "", false + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", "", false + } + host := strings.ToLower(u.Hostname()) + if host != "luma.com" && host != "www.luma.com" && host != "lu.ma" { + return "", "", false + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) != 2 || parts[0] != "check-in" || parts[1] == "" { + return "", "", false + } + pk = u.Query().Get("pk") + if pk == "" { + return "", "", false + } + return parts[1], pk, true +} diff --git a/plugins/inputs/qr_scanner/url_parse_test.go b/plugins/inputs/qr_scanner/url_parse_test.go new file mode 100644 index 0000000000..4a15995a67 --- /dev/null +++ b/plugins/inputs/qr_scanner/url_parse_test.go @@ -0,0 +1,118 @@ +package qr_scanner_test + +import ( + "testing" + + "github.com/openmind/om1/plugins/inputs/qr_scanner" +) + +func TestParseLumaCheckinURL(t *testing.T) { + cases := []struct { + name string + input string + wantEvent string + wantPK string + wantOK bool + }{ + { + name: "happy path luma.com", + input: "https://luma.com/check-in/evt-abc?pk=g-12345", + wantEvent: "evt-abc", + wantPK: "g-12345", + wantOK: true, + }, + { + name: "lu.ma short host", + input: "https://lu.ma/check-in/evt-abc?pk=tk_xyz", + wantEvent: "evt-abc", + wantPK: "tk_xyz", + wantOK: true, + }, + { + name: "www.luma.com", + input: "https://www.luma.com/check-in/evt-abc?pk=g-12345", + wantEvent: "evt-abc", + wantPK: "g-12345", + wantOK: true, + }, + { + name: "trailing slash on path", + input: "https://luma.com/check-in/evt-abc/?pk=g-12345", + wantEvent: "evt-abc", + wantPK: "g-12345", + wantOK: true, + }, + { + name: "http scheme accepted", + input: "http://luma.com/check-in/evt-abc?pk=g-12345", + wantEvent: "evt-abc", + wantPK: "g-12345", + wantOK: true, + }, + { + name: "percent-encoded pk", + input: "https://luma.com/check-in/evt-abc?pk=g%2D12345", + wantEvent: "evt-abc", + wantPK: "g-12345", + wantOK: true, + }, + { + name: "missing pk", + input: "https://luma.com/check-in/evt-abc", + wantOK: false, + }, + { + name: "empty pk", + input: "https://luma.com/check-in/evt-abc?pk=", + wantOK: false, + }, + { + name: "wrong host", + input: "https://example.com/check-in/evt-abc?pk=g-12345", + wantOK: false, + }, + { + name: "wrong path", + input: "https://luma.com/event/evt-abc?pk=g-12345", + wantOK: false, + }, + { + name: "extra path segment", + input: "https://luma.com/check-in/evt-abc/extra?pk=g-12345", + wantOK: false, + }, + { + name: "missing event id", + input: "https://luma.com/check-in/?pk=g-12345", + wantOK: false, + }, + { + name: "non-url garbage", + input: "hello world", + wantOK: false, + }, + { + name: "ftp scheme rejected", + input: "ftp://luma.com/check-in/evt-abc?pk=g-12345", + wantOK: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + gotEvent, gotPK, gotOK := qr_scanner.ParseLumaCheckinURL(tc.input) + if gotOK != tc.wantOK { + t.Fatalf("ok mismatch: got %v want %v", gotOK, tc.wantOK) + } + if !tc.wantOK { + return + } + if gotEvent != tc.wantEvent { + t.Errorf("event: got %q want %q", gotEvent, tc.wantEvent) + } + if gotPK != tc.wantPK { + t.Errorf("pk: got %q want %q", gotPK, tc.wantPK) + } + }) + } +} From d5448b0d0f9c0fb0d15a5370b5dff8c6f5bb7727 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Wed, 10 Jun 2026 16:28:35 -0700 Subject: [PATCH 03/40] update config --- config/luma_checkin_local.json5 | 2 +- internal/providers/vlm/video_stream.go | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 index 3c17bcaf93..f50fee0fb6 100644 --- a/config/luma_checkin_local.json5 +++ b/config/luma_checkin_local.json5 @@ -7,7 +7,7 @@ version: "v1.0.0", system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", agent_inputs: [ { - type: "VLMVila", + type: "VLMGemini", }, ], cortex_llm: { diff --git a/internal/providers/vlm/video_stream.go b/internal/providers/vlm/video_stream.go index ee83eff3c1..2bc7cf77bd 100644 --- a/internal/providers/vlm/video_stream.go +++ b/internal/providers/vlm/video_stream.go @@ -137,14 +137,27 @@ func (v *VideoStream) cameraInput() string { // ffmpegArgs constructs the ffmpeg command-line arguments for the configured stream. func (v *VideoStream) ffmpegArgs(cam string) []string { - inputFormat := "v4l2" if runtime.GOOS == "darwin" { - inputFormat = "avfoundation" + // avfoundation rejects yuv420p (its default) and requires an explicit pixel + // format. yuyv422 is universally supported by built-in Mac cameras. + return []string{ + "-loglevel", "error", + "-f", "avfoundation", + "-framerate", strconv.Itoa(v.cfg.FPS), + "-pixel_format", "yuyv422", + "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height), + "-i", cam, + "-an", + "-c:v", "mjpeg", + "-qscale:v", strconv.Itoa(jpegQScale(v.cfg.JPEGQuality)), + "-f", "image2pipe", + "pipe:1", + } } return []string{ "-loglevel", "error", - "-f", inputFormat, + "-f", "v4l2", "-framerate", strconv.Itoa(v.cfg.FPS), "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height), "-i", cam, From 9c7f0fe11432f4bb7337b9f6a777d58097432d46 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Wed, 10 Jun 2026 17:18:43 -0700 Subject: [PATCH 04/40] update input plugin --- config/luma_checkin_local.json5 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 index f50fee0fb6..adae474aed 100644 --- a/config/luma_checkin_local.json5 +++ b/config/luma_checkin_local.json5 @@ -7,7 +7,16 @@ version: "v1.0.0", system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", agent_inputs: [ { - type: "VLMGemini", + type: "QRScanner", + config: { + camera_index: 0, + capture_fps: 15, + decode_fps: 5, + resolution_width: 640, + resolution_height: 480, + jpeg_quality: 60, + dedupe_window_seconds: 30, + }, }, ], cortex_llm: { From 1f32c6996cea32616e392b27e015a960fa8f5c5d Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 11:18:50 -0700 Subject: [PATCH 05/40] fix mac camera issue --- config/conversation.json5 | 7 ++- internal/providers/vlm/video_stream.go | 66 ++++++++++++++++---------- internal/providers/vlm/vlm_test.go | 6 +-- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/config/conversation.json5 b/config/conversation.json5 index a412c3a259..ddbafa548b 100644 --- a/config/conversation.json5 +++ b/config/conversation.json5 @@ -21,7 +21,12 @@ }, }, { - type: "VLMVila", + type: "VLMGemini", + config: { + resolution_width: 1280, + resolution_height: 720, + fps: 30, + }, } ], cortex_llm: { diff --git a/internal/providers/vlm/video_stream.go b/internal/providers/vlm/video_stream.go index 2bc7cf77bd..fb87be6e08 100644 --- a/internal/providers/vlm/video_stream.go +++ b/internal/providers/vlm/video_stream.go @@ -1,8 +1,10 @@ package vlm import ( + "bufio" "context" "fmt" + "io" "os/exec" "runtime" "strconv" @@ -16,8 +18,6 @@ import ( const ( defaultFPS = 30 - defaultWidth = 640 - defaultHeight = 480 defaultJPEGQuality = 30 cameraRetryDelay = 2 * time.Second ) @@ -40,18 +40,6 @@ type VideoStream struct { } func NewVideoStream(cfg VideoStreamConfig) *VideoStream { - if cfg.FPS <= 0 { - cfg.FPS = defaultFPS - } - - if cfg.Width <= 0 { - cfg.Width = defaultWidth - } - - if cfg.Height <= 0 { - cfg.Height = defaultHeight - } - if cfg.JPEGQuality <= 0 { cfg.JPEGQuality = defaultJPEGQuality } @@ -103,10 +91,17 @@ func (v *VideoStream) stream(ctx context.Context, cam string) error { return fmt.Errorf("stdout pipe: %w", err) } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + if err := cmd.Start(); err != nil { return fmt.Errorf("start ffmpeg: %w", err) } + go v.logFFmpegStderr(stderr) + defer func() { if cmd.Process != nil { _ = cmd.Process.Kill() @@ -126,6 +121,13 @@ func (v *VideoStream) stream(ctx context.Context, cam string) error { }) } +func (v *VideoStream) logFFmpegStderr(r io.ReadCloser) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + v.log.Warn("VideoStream: ffmpeg", zap.String("stderr", scanner.Text())) + } +} + // cameraInput returns the ffmpeg input specifier for the configured camera device. func (v *VideoStream) cameraInput() string { if runtime.GOOS == "darwin" { @@ -138,33 +140,47 @@ func (v *VideoStream) cameraInput() string { // ffmpegArgs constructs the ffmpeg command-line arguments for the configured stream. func (v *VideoStream) ffmpegArgs(cam string) []string { if runtime.GOOS == "darwin" { - // avfoundation rejects yuv420p (its default) and requires an explicit pixel - // format. yuyv422 is universally supported by built-in Mac cameras. - return []string{ + // avfoundation pairs framerate to the chosen video size, so only emit + // -framerate when -video_size is also set; otherwise ffmpeg picks a + // default size whose mode list may not include the requested rate. + // Pixel format is left unset so avfoundation picks the camera's native + // format (e.g. nv12 on Apple Silicon / Continuity cameras). + args := []string{ "-loglevel", "error", "-f", "avfoundation", - "-framerate", strconv.Itoa(v.cfg.FPS), - "-pixel_format", "yuyv422", - "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height), + } + if v.cfg.Width > 0 && v.cfg.Height > 0 { + args = append(args, "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height)) + if v.cfg.FPS > 0 { + args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) + } + } + return append(args, "-i", cam, "-an", "-c:v", "mjpeg", "-qscale:v", strconv.Itoa(jpegQScale(v.cfg.JPEGQuality)), "-f", "image2pipe", "pipe:1", - } + ) } - return []string{ + args := []string{ "-loglevel", "error", "-f", "v4l2", - "-framerate", strconv.Itoa(v.cfg.FPS), - "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height), + } + if v.cfg.Width > 0 && v.cfg.Height > 0 { + args = append(args, "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height)) + if v.cfg.FPS > 0 { + args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) + } + } + return append(args, "-i", cam, "-an", "-c:v", "mjpeg", "-qscale:v", strconv.Itoa(jpegQScale(v.cfg.JPEGQuality)), "-f", "image2pipe", "pipe:1", - } + ) } diff --git a/internal/providers/vlm/vlm_test.go b/internal/providers/vlm/vlm_test.go index da72652621..261147564e 100644 --- a/internal/providers/vlm/vlm_test.go +++ b/internal/providers/vlm/vlm_test.go @@ -141,9 +141,9 @@ func TestParseAVFoundationDevices(t *testing.T) { func TestNewVideoStreamDefaults(t *testing.T) { v := NewVideoStream(VideoStreamConfig{}) - assert.Equal(t, defaultFPS, v.cfg.FPS) - assert.Equal(t, defaultWidth, v.cfg.Width) - assert.Equal(t, defaultHeight, v.cfg.Height) + assert.Equal(t, 0, v.cfg.FPS) + assert.Equal(t, 0, v.cfg.Width) + assert.Equal(t, 0, v.cfg.Height) assert.Equal(t, defaultJPEGQuality, v.cfg.JPEGQuality) } From 597ab5e76cdca5704276d116c1c318aad15bf70d Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 12:24:43 -0700 Subject: [PATCH 06/40] add new config and rtsp for qr scanner --- config/luma_checkin_go2.json5 | 44 +++++++++++++++++++ plugins/inputs/qr_scanner/qr_scanner.go | 56 +++++++++++++++++++------ 2 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 config/luma_checkin_go2.json5 diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 new file mode 100644 index 0000000000..4c72be9fe7 --- /dev/null +++ b/config/luma_checkin_go2.json5 @@ -0,0 +1,44 @@ +{ +version: "v1.0.0", + hertz: 0.2, + name: "luma_checkin_local", + api_key: "${OM_API_KEY:-openmind_free}", + system_prompt_base: "You are a test harness for the Luma check-in flow. The Vision input emits a line of the form 'qr_scan: pk= event=' when it sees a Luma check-in QR code, or 'none' when it does not. When you see 'qr_scan: pk=...', immediately call the luma_checkin action with that pk value. When you see 'none', do nothing. Do not narrate.", + system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", + agent_inputs: [ + { + type: "QRScannerRTSP", + config: { + camera_index: 0, + capture_fps: 15, + decode_fps: 5, + dedupe_window_seconds: 30, + }, + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 4, + }, + }, + agent_actions: [ + { + name: "luma_checkin", + llm_label: "luma_checkin", + connector: "api", + config: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + request_timeout_seconds: 5, + tts: { + provider: "elevenlabs", + elevenlabs_api_key: "${ELEVENLABS_API_KEY}", + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + }, + ], +} diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go index 0e6caef4e1..f0e7a00e6e 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -17,6 +17,7 @@ import ( const ( scannerName = "QRScanner" + scannerRTSPName = "QRScannerRTSP" scannerDescriptor = "QR Scanner" scannerMaxMessages = 8 scanChannelBuffer = 4 @@ -24,11 +25,13 @@ const ( func init() { inputs.Register(scannerName, NewQRScanner) + inputs.Register(scannerRTSPName, NewQRScannerRTSP) } // Config holds the JSON configuration for the QRScanner input plugin. type Config struct { CameraIndex int `json:"camera_index"` + RTSPURL string `json:"rtsp_url"` CaptureFPS int `json:"capture_fps"` DecodeFPS int `json:"decode_fps"` Width int `json:"resolution_width"` @@ -43,6 +46,7 @@ type frameSource interface { } type sensor struct { + name string cfg Config log *zap.Logger source frameSource @@ -54,18 +58,10 @@ type sensor struct { cancel context.CancelFunc } -// NewQRScanner constructs a QRScanner sensor from the decoded config map. +// NewQRScanner constructs a camera-backed QRScanner sensor. func NewQRScanner(configMap map[string]any) (inputs.Sensor, error) { cfg := parseConfig(configMap) - log := logger.Get().Named(scannerName) - log.Info("initializing", - zap.Int("camera_index", cfg.CameraIndex), - zap.Int("capture_fps", cfg.CaptureFPS), - zap.Int("decode_fps", cfg.DecodeFPS), - zap.Int("width", cfg.Width), - zap.Int("height", cfg.Height), - zap.Float64("dedupe_window_seconds", cfg.DedupeWindowSeconds), - ) + log := newLogger(scannerName, cfg) source := video.NewVideoStream(video.VideoStreamConfig{ DeviceIndex: cfg.CameraIndex, @@ -74,14 +70,50 @@ func NewQRScanner(configMap map[string]any) (inputs.Sensor, error) { Height: cfg.Height, JPEGQuality: cfg.JPEGQuality, }) + return newSensor(cfg, log, source), nil +} + +// NewQRScannerRTSP constructs an RTSP-backed QRScanner sensor. +func NewQRScannerRTSP(configMap map[string]any) (inputs.Sensor, error) { + cfg := parseConfig(configMap) + if cfg.RTSPURL == "" { + cfg.RTSPURL = "rtsp://localhost:8554/top_camera_raw" + } + log := newLogger(scannerRTSPName, cfg) + + source := video.NewVideoRTSPStream(video.VideoRTSPStreamConfig{ + RTSPURL: cfg.RTSPURL, + FPS: cfg.CaptureFPS, + Width: cfg.Width, + Height: cfg.Height, + JPEGQuality: cfg.JPEGQuality, + }) + return newSensor(cfg, log, source), nil +} +func newLogger(name string, cfg Config) *zap.Logger { + log := logger.Get().Named(name) + log.Info("initializing", + zap.Int("camera_index", cfg.CameraIndex), + zap.String("rtsp_url", cfg.RTSPURL), + zap.Int("capture_fps", cfg.CaptureFPS), + zap.Int("decode_fps", cfg.DecodeFPS), + zap.Int("width", cfg.Width), + zap.Int("height", cfg.Height), + zap.Float64("dedupe_window_seconds", cfg.DedupeWindowSeconds), + ) + return log +} + +func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { window := time.Duration(cfg.DedupeWindowSeconds * float64(time.Second)) return &sensor{ + name: log.Name(), cfg: cfg, log: log, source: source, debouncer: newDebouncer(window), - }, nil + } } func parseConfig(configMap map[string]any) Config { @@ -209,7 +241,7 @@ func (s *sensor) FormattedLatestBuffer() string { result := fmt.Sprintf("\n%s: '%s'\n", scannerDescriptor, latest.Message) ts := time.Unix(0, int64(latest.Timestamp*1e9)) - providers.IO().AddInput(scannerName, latest.Message, ts) + providers.IO().AddInput(s.name, latest.Message, ts) s.messages = nil return result From 5a190261aba84b945638280bdfd543a1f00d9143 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 16:58:18 -0700 Subject: [PATCH 07/40] updated qr scanner logic and added a new luma provider --- internal/providers/luma/client.go | 209 ++++++++++++++++++++++++ plugins/inputs/qr_scanner/qr_scanner.go | 117 +++++++++++-- 2 files changed, 316 insertions(+), 10 deletions(-) create mode 100644 internal/providers/luma/client.go diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go new file mode 100644 index 0000000000..7671a4d5fc --- /dev/null +++ b/internal/providers/luma/client.go @@ -0,0 +1,209 @@ +// Package luma provides a thin client for Luma's public API. Currently it only +// exposes /v1/event/get-guest, which is enough to look up a guest by the pk +// embedded in a check-in QR code and surface their name to the rest of OM1. +package luma + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/openmind/om1/internal/httpclient" +) + +const ( + DefaultBaseURL = "https://public-api.luma.com" + GetGuestPath = "/v1/event/get-guest" +) + +var ( + ErrNotFound = errors.New("luma: guest not found") + ErrUnauthorized = errors.New("luma: unauthorized") +) + +// Guest is a flattened view of the guest record returned by Luma's +// /v1/event/get-guest. Identity fields use the user_-prefixed names from the +// public API schema. Unknown fields are ignored. +type Guest struct { + APIID string `json:"api_id"` + UserAPIID string `json:"user_api_id"` + UserName string `json:"user_name"` + UserFirstName string `json:"user_first_name"` + UserLastName string `json:"user_last_name"` + UserEmail string `json:"user_email"` + EventAPIID string `json:"event_api_id"` + CheckedInAt string `json:"checked_in_at"` + ApprovalStatus string `json:"approval_status"` +} + +// guestEnvelope handles Luma's nested response shape: {"guest": {...}, "event": {...}}. +// Some endpoints return the guest object directly; we try the envelope first. +type guestEnvelope struct { + Guest *Guest `json:"guest"` + Event *struct { + APIID string `json:"api_id"` + } `json:"event"` +} + +// HTTPDoer is satisfied by *http.Client; abstracted for tests. +type HTTPDoer interface { + Do(req *http.Request) (*http.Response, error) +} + +type Client struct { + baseURL string + apiKey string + eventAPIID string + http HTTPDoer + timeout time.Duration +} + +func NewClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *Client { + if baseURL == "" { + baseURL = DefaultBaseURL + } + if timeout <= 0 { + timeout = 5 * time.Second + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + apiKey: apiKey, + eventAPIID: eventAPIID, + http: httpclient.Default(), + timeout: timeout, + } +} + +// SetHTTPDoer overrides the underlying http client. Intended for tests. +func (c *Client) SetHTTPDoer(d HTTPDoer) { c.http = d } + +// EventAPIID returns the configured event id. +func (c *Client) EventAPIID() string { return c.eventAPIID } + +// GetGuest fetches the guest record for the given key (g- or ticket key). +// Luma's get-guest requires both event_id (the configured event) and id (the +// guest/ticket key from the QR). +func (c *Client) GetGuest(ctx context.Context, pk string) (*Guest, error) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + q := url.Values{ + "event_id": []string{c.eventAPIID}, + "id": []string{pk}, + } + endpoint := c.baseURL + GetGuestPath + "?" + q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + c.setAuth(req) + + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, ErrNotFound + case http.StatusUnauthorized, http.StatusForbidden: + return nil, ErrUnauthorized + default: + return nil, fmt.Errorf("luma get-guest %d: %s", resp.StatusCode, string(body)) + } + + var env guestEnvelope + if err := json.Unmarshal(body, &env); err == nil && env.Guest != nil { + g := env.Guest + if g.EventAPIID == "" && env.Event != nil { + g.EventAPIID = env.Event.APIID + } + return g, nil + } + var bare Guest + if err := json.Unmarshal(body, &bare); err != nil { + return nil, fmt.Errorf("luma get-guest decode: %w", err) + } + return &bare, nil +} + +func (c *Client) setAuth(req *http.Request) { + req.Header.Set("x-luma-api-key", c.apiKey) + req.Header.Set("Accept", "application/json") +} + +// do issues req with one retry on 429 after a short backoff. +func (c *Client) do(req *http.Request) (*http.Response, error) { + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusTooManyRequests { + return resp, nil + } + + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + t := time.NewTimer(500 * time.Millisecond) + defer t.Stop() + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-t.C: + } + + if req.GetBody != nil { + body, berr := req.GetBody() + if berr != nil { + return nil, berr + } + req.Body = body + } + return c.http.Do(req) +} + +// FirstName picks the best available first-name field from a Guest record. +// Falls back to "friend" when nothing usable is set. +func FirstName(g *Guest) string { + if g == nil { + return "friend" + } + if g.UserFirstName != "" { + return g.UserFirstName + } + if g.UserName != "" { + if parts := strings.Fields(g.UserName); len(parts) > 0 { + return parts[0] + } + } + return "friend" +} + +// FormatGreeting interpolates {first_name}, {last_name}, {name}, {email} into +// template using the guest's fields. +func FormatGreeting(template string, g *Guest) string { + if g == nil { + return template + } + name := g.UserName + if name == "" { + name = FirstName(g) + } + r := strings.NewReplacer( + "{first_name}", FirstName(g), + "{last_name}", g.UserLastName, + "{name}", name, + "{email}", g.UserEmail, + ) + return r.Replace(template) +} diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go index f0e7a00e6e..7a7cc61a58 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -3,7 +3,9 @@ package qr_scanner import ( "context" "encoding/json" + "errors" "fmt" + "strings" "sync" "time" @@ -12,6 +14,7 @@ import ( "github.com/openmind/om1/internal/inputs" "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" + "github.com/openmind/om1/internal/providers/luma" video "github.com/openmind/om1/internal/providers/vlm" ) @@ -21,6 +24,9 @@ const ( scannerDescriptor = "QR Scanner" scannerMaxMessages = 8 scanChannelBuffer = 4 + + defaultGreetingTemplate = "Welcome, {first_name}!" + defaultLumaTimeout = 4 * time.Second ) func init() { @@ -28,16 +34,29 @@ func init() { inputs.Register(scannerRTSPName, NewQRScannerRTSP) } +// LumaConfig configures an optional guest lookup against Luma. When set, every +// scan that passes dedupe is enriched with the guest's name and a personalized +// greeting before being emitted, so downstream LLM/speak doesn't have to +// hallucinate the name. +type LumaConfig struct { + APIKey string `json:"api_key"` + BaseURL string `json:"base_url"` + EventAPIID string `json:"event_api_id"` + GreetingTemplate string `json:"greeting_template"` + RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` +} + // Config holds the JSON configuration for the QRScanner input plugin. type Config struct { - CameraIndex int `json:"camera_index"` - RTSPURL string `json:"rtsp_url"` - CaptureFPS int `json:"capture_fps"` - DecodeFPS int `json:"decode_fps"` - Width int `json:"resolution_width"` - Height int `json:"resolution_height"` - JPEGQuality int `json:"jpeg_quality"` - DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` + CameraIndex int `json:"camera_index"` + RTSPURL string `json:"rtsp_url"` + CaptureFPS int `json:"capture_fps"` + DecodeFPS int `json:"decode_fps"` + Width int `json:"resolution_width"` + Height int `json:"resolution_height"` + JPEGQuality int `json:"jpeg_quality"` + DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` + Luma *LumaConfig `json:"luma"` } type frameSource interface { @@ -45,6 +64,12 @@ type frameSource interface { Stop() } +// guestLookup is the slice of luma.Client behavior the scanner uses; +// abstracted for tests. +type guestLookup interface { + GetGuest(ctx context.Context, pk string) (*luma.Guest, error) +} + type sensor struct { name string cfg Config @@ -52,6 +77,11 @@ type sensor struct { source frameSource debouncer *debouncer + luma guestLookup + greetingTmpl string + lumaTimeout time.Duration + expectedEventID string + mu sync.Mutex messages []inputs.Message stopped bool @@ -107,13 +137,33 @@ func newLogger(name string, cfg Config) *zap.Logger { func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { window := time.Duration(cfg.DedupeWindowSeconds * float64(time.Second)) - return &sensor{ + s := &sensor{ name: log.Name(), cfg: cfg, log: log, source: source, debouncer: newDebouncer(window), } + + if cfg.Luma != nil && cfg.Luma.APIKey != "" && cfg.Luma.EventAPIID != "" { + timeout := time.Duration(cfg.Luma.RequestTimeoutSeconds * float64(time.Second)) + if timeout <= 0 { + timeout = defaultLumaTimeout + } + s.luma = luma.NewClient(cfg.Luma.BaseURL, cfg.Luma.APIKey, cfg.Luma.EventAPIID, timeout) + s.lumaTimeout = timeout + s.expectedEventID = cfg.Luma.EventAPIID + s.greetingTmpl = cfg.Luma.GreetingTemplate + if s.greetingTmpl == "" { + s.greetingTmpl = defaultGreetingTemplate + } + log.Info("luma lookup enabled", + zap.String("event_api_id", cfg.Luma.EventAPIID), + zap.Duration("timeout", timeout), + ) + } + + return s } func parseConfig(configMap map[string]any) Config { @@ -192,7 +242,7 @@ func (s *sensor) Listen(ctx context.Context) (<-chan any, error) { continue } - msg := fmt.Sprintf("qr_scan: pk=%s event=%s", pk, eventID) + msg := s.formatScanMessage(ctx, pk, eventID) s.log.Info("emitted scan", zap.String("pk", pk), zap.String("event", eventID)) select { case out <- msg: @@ -273,3 +323,50 @@ func truncate(s string, n int) string { } return s[:n] + "..." } + +// formatScanMessage builds the observation string emitted into the LLM prompt. +// When a Luma client is configured, it does a guest lookup and embeds the +// resolved name and personalized greeting directly so the LLM doesn't have to +// hallucinate them when relaying to speak. On lookup failure or auth/event +// mismatch, the message falls back to a bare scan or a known failure marker +// so prompt logic can branch. +func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) string { + if s.luma == nil { + return fmt.Sprintf("qr_scan: pk=%s event=%s", pk, eventID) + } + + if s.expectedEventID != "" && eventID != "" && eventID != s.expectedEventID { + s.log.Info("scan event mismatch", + zap.String("expected", s.expectedEventID), + zap.String("got", eventID), + ) + return fmt.Sprintf("qr_scan_failed: pk=%s reason=event_mismatch", pk) + } + + lookupCtx, cancel := context.WithTimeout(ctx, s.lumaTimeout) + defer cancel() + + guest, err := s.luma.GetGuest(lookupCtx, pk) + if err != nil { + switch { + case errors.Is(err, luma.ErrNotFound): + s.log.Info("luma lookup: not found", zap.String("pk", pk)) + return fmt.Sprintf("qr_scan_failed: pk=%s reason=guest_not_registered", pk) + case errors.Is(err, luma.ErrUnauthorized): + s.log.Error("luma lookup: unauthorized") + return fmt.Sprintf("qr_scan_failed: pk=%s reason=luma_auth", pk) + default: + s.log.Warn("luma lookup failed", zap.String("pk", pk), zap.Error(err)) + return fmt.Sprintf("qr_scan_failed: pk=%s reason=lookup_error", pk) + } + } + if guest == nil { + return fmt.Sprintf("qr_scan_failed: pk=%s reason=empty_response", pk) + } + + name := luma.FirstName(guest) + greeting := luma.FormatGreeting(s.greetingTmpl, guest) + greeting = strings.ReplaceAll(greeting, `"`, `'`) + s.log.Info("luma lookup ok", zap.String("pk", pk), zap.String("name", name)) + return fmt.Sprintf(`qr_scan: name=%s greeting="%s"`, name, greeting) +} From bf00509f322f48e5b3a37358900e1428aa114e24 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 16:59:17 -0700 Subject: [PATCH 08/40] deleted old luma action plugin --- plugins/actions/luma_checkin/export_test.go | 47 --- plugins/actions/luma_checkin/luma_checkin.go | 291 ------------------ .../actions/luma_checkin/luma_checkin_test.go | 196 ------------ plugins/actions/luma_checkin/luma_client.go | 198 ------------ .../actions/luma_checkin/luma_client_test.go | 155 ---------- 5 files changed, 887 deletions(-) delete mode 100644 plugins/actions/luma_checkin/export_test.go delete mode 100644 plugins/actions/luma_checkin/luma_checkin.go delete mode 100644 plugins/actions/luma_checkin/luma_checkin_test.go delete mode 100644 plugins/actions/luma_checkin/luma_client.go delete mode 100644 plugins/actions/luma_checkin/luma_client_test.go diff --git a/plugins/actions/luma_checkin/export_test.go b/plugins/actions/luma_checkin/export_test.go deleted file mode 100644 index 4697e74ee0..0000000000 --- a/plugins/actions/luma_checkin/export_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package luma_checkin - -import ( - "context" - "time" - - "go.uber.org/zap" -) - -const ( - GetGuestPath = getGuestPath - UpdateGuestStatusPath = updateGuestStatusPath - DefaultGreetingTemplate = defaultGreetingTemplate - DefaultStatus = defaultStatus -) - -var ( - ErrLumaNotFound = errLumaNotFound - ErrLumaUnauthorized = errLumaUnauthorized -) - -type LumaClient = lumaClient - -func NewLumaClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *LumaClient { - return newLumaClient(baseURL, apiKey, eventAPIID, timeout) -} - -func FirstNameFor(g *Guest) string { return firstNameFor(g) } -func FormatGreeting(tmpl string, g *Guest) string { return formatGreeting(tmpl, g) } - -type GuestLookup = guestLookup -type TTSPlayer = ttsPlayer - -type TestConnector struct{ *connector } - -func NewTestConnector(client GuestLookup, cfg Config, tts TTSPlayer) *TestConnector { - return &TestConnector{&connector{ - log: zap.NewNop(), - cfg: cfg, - client: client, - tts: tts, - }} -} - -func (c *TestConnector) Connect(ctx context.Context, in map[string]any) (any, error) { - return c.connector.Connect(ctx, in) -} diff --git a/plugins/actions/luma_checkin/luma_checkin.go b/plugins/actions/luma_checkin/luma_checkin.go deleted file mode 100644 index ce8736cac1..0000000000 --- a/plugins/actions/luma_checkin/luma_checkin.go +++ /dev/null @@ -1,291 +0,0 @@ -package luma_checkin - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "sync" - "time" - - "go.uber.org/zap" - - "github.com/openmind/om1/internal/actions" - "github.com/openmind/om1/internal/logger" - "github.com/openmind/om1/internal/providers/tts" -) - -const ( - defaultGreetingTemplate = "Welcome, {first_name}! Glad you made it." - defaultStatus = "checked_in" - sideEffectTimeout = 5 * time.Second -) - -// LumaCheckinInput is the LLM-visible argument schema. The event is configured -// per-deployment; the LLM only supplies the pk parsed from the QR code. -type LumaCheckinInput struct { - Pk string `json:"pk" description:"The Luma guest or ticket key (pk parameter from check-in QR code)"` -} - -// ttsConfig configures which TTS provider to use for the welcome utterance. -// Both Kokoro and ElevenLabs providers are sync.Once singletons, so this -// instance is shared with the speak action plugin if both are configured; -// the first caller's config wins. -type ttsConfig struct { - Provider string `json:"provider"` - APIKey string `json:"api_key"` - ElevenLabsAPIKey string `json:"elevenlabs_api_key"` - BaseURL string `json:"base_url"` - VoiceID string `json:"voice_id"` - ModelID string `json:"model_id"` - OutputFormat string `json:"output_format"` - Rate int `json:"rate"` -} - -// Config holds the plugin configuration decoded from JSON5. -type Config struct { - APIKey string `json:"api_key"` - BaseURL string `json:"base_url"` - EventAPIID string `json:"event_api_id"` - GreetingTemplate string `json:"greeting_template"` - RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` - StatusValue string `json:"status_value"` - TTS ttsConfig `json:"tts"` -} - -// ttsPlayer is the minimal interface luma_checkin needs from a TTS provider. -type ttsPlayer interface { - AddText(text string) -} - -// guestLookup is the interface luma_checkin uses against Luma; abstracted for tests. -type guestLookup interface { - GetGuest(ctx context.Context, pk string) (*Guest, error) - UpdateGuestStatus(ctx context.Context, eventAPIID, guestAPIID, status string) error -} - -type connector struct { - log *zap.Logger - cfg Config - client guestLookup - tts ttsPlayer - - mu sync.Mutex - stopped bool -} - -func init() { - actions.RegisterInterface( - "luma_checkin", - "Action interface for checking guests into a Luma event. "+ - "Looks up the guest via the Luma external check-in API, validates the configured event, "+ - "flips status to checked_in, and speaks a personalized welcome via the configured TTS provider. "+ - "For arm gestures and facial expressions, call unitree_g1_arm and emotion alongside this action.", - LumaCheckinInput{}, - ) - actions.Register("luma_checkin/api", NewLumaCheckinConnector) -} - -// NewLumaCheckinConnector constructs a connector from the decoded config map. -func NewLumaCheckinConnector(cfgMap map[string]any) (actions.Connector, error) { - cfg, err := parseConfig(cfgMap) - if err != nil { - return nil, err - } - log := logger.Get().Named("luma_checkin/api") - - timeout := time.Duration(cfg.RequestTimeoutSeconds * float64(time.Second)) - c := &connector{ - log: log, - cfg: cfg, - client: newLumaClient(cfg.BaseURL, cfg.APIKey, cfg.EventAPIID, timeout), - tts: buildTTS(cfg.TTS, log), - } - - log.Info("initialized", - zap.String("event_api_id", cfg.EventAPIID), - zap.String("tts_provider", cfg.TTS.Provider), - ) - return c, nil -} - -func parseConfig(cfgMap map[string]any) (Config, error) { - var cfg Config - if b, err := json.Marshal(cfgMap); err == nil { - _ = json.Unmarshal(b, &cfg) - } - if cfg.APIKey == "" { - return cfg, fmt.Errorf("luma_checkin: api_key is required") - } - if cfg.EventAPIID == "" { - return cfg, fmt.Errorf("luma_checkin: event_api_id is required") - } - if cfg.GreetingTemplate == "" { - cfg.GreetingTemplate = defaultGreetingTemplate - } - if cfg.StatusValue == "" { - cfg.StatusValue = defaultStatus - } - if cfg.RequestTimeoutSeconds <= 0 { - cfg.RequestTimeoutSeconds = 5 - } - return cfg, nil -} - -func buildTTS(cfg ttsConfig, log *zap.Logger) ttsPlayer { - switch strings.ToLower(cfg.Provider) { - case "kokoro": - return tts.Kokoro(tts.KokoroConfig{ - BaseURL: orDefault(cfg.BaseURL, tts.DefaultKokoroBaseURL), - APIKey: cfg.APIKey, - VoiceID: orDefault(cfg.VoiceID, tts.DefaultKokoroVoiceID), - ModelID: orDefault(cfg.ModelID, tts.DefaultKokoroModelID), - OutputFormat: orDefault(cfg.OutputFormat, tts.DefaultKokoroOutputFormat), - Rate: orDefaultInt(cfg.Rate, tts.DefaultKokoroRate), - }, log) - case "elevenlabs": - return tts.ElevenLabs(tts.ElevenLabsConfig{ - APIKey: cfg.APIKey, - ElevenLabsAPIKey: cfg.ElevenLabsAPIKey, - VoiceID: orDefault(cfg.VoiceID, tts.DefaultVoiceID), - ModelID: orDefault(cfg.ModelID, tts.DefaultModelID), - OutputFormat: orDefault(cfg.OutputFormat, tts.DefaultOutputFormat), - Rate: orDefaultInt(cfg.Rate, tts.DefaultRate), - }, log) - case "": - return nil - default: - log.Warn("luma_checkin: unknown tts.provider", zap.String("provider", cfg.Provider)) - return nil - } -} - -// Connect performs the check-in: GetGuest → validate event → fan out the -// status update + TTS welcome → return status string. Arm wave and emotion -// are intentionally not handled here; configure unitree_g1_arm and emotion -// actions and let the LLM call them concurrently. -func (c *connector) Connect(ctx context.Context, input actions.Input) (actions.Output, error) { - args, ok := input.(map[string]any) - if !ok { - return nil, fmt.Errorf("luma_checkin: unexpected input type %T", input) - } - pk := strings.TrimSpace(stringField(args, "pk")) - if pk == "" { - return nil, fmt.Errorf("luma_checkin: pk required") - } - - guest, err := c.client.GetGuest(ctx, pk) - if err != nil { - switch { - case errors.Is(err, errLumaNotFound): - c.log.Info("guest not found", zap.String("pk", pk)) - return "checkin_failed: guest not registered for this event", nil - case errors.Is(err, errLumaUnauthorized): - c.log.Error("luma auth failed") - return nil, fmt.Errorf("luma api auth failed") - default: - c.log.Warn("get-guest failed", zap.Error(err)) - return nil, err - } - } - if guest == nil { - return "checkin_failed: empty guest response", nil - } - if guest.EventAPIID != "" && guest.EventAPIID != c.cfg.EventAPIID { - c.log.Info("event mismatch", - zap.String("expected", c.cfg.EventAPIID), - zap.String("got", guest.EventAPIID), - ) - return "checkin_failed: qr is for a different event", nil - } - - displayName := firstNameFor(guest) - - sideCtx, cancel := context.WithTimeout(ctx, sideEffectTimeout) - defer cancel() - - var wg sync.WaitGroup - var statusErr error - var statusMu sync.Mutex - - if guest.APIID != "" { - wg.Add(1) - go func() { - defer wg.Done() - if err := c.client.UpdateGuestStatus(sideCtx, c.cfg.EventAPIID, guest.APIID, c.cfg.StatusValue); err != nil { - c.log.Warn("update-guest-status failed", zap.Error(err)) - statusMu.Lock() - statusErr = err - statusMu.Unlock() - } - }() - } - - if c.tts != nil { - c.tts.AddText(formatGreeting(c.cfg.GreetingTemplate, guest)) - } - - wg.Wait() - - statusMu.Lock() - failed := statusErr != nil - statusMu.Unlock() - - c.log.Info("checked in", zap.String("pk", pk), zap.String("name", displayName), zap.Bool("status_update_failed", failed)) - if failed { - return fmt.Sprintf("checked_in: %s (status update failed, logged)", displayName), nil - } - return fmt.Sprintf("checked_in: %s", displayName), nil -} - -func (c *connector) Tick(ctx context.Context) { - <-ctx.Done() -} - -func (c *connector) Stop() { - c.mu.Lock() - defer c.mu.Unlock() - c.stopped = true -} - -func stringField(m map[string]any, key string) string { - v, _ := m[key].(string) - return v -} - -func firstNameFor(g *Guest) string { - if g.FirstName != "" { - return g.FirstName - } - if g.Name != "" { - if parts := strings.Fields(g.Name); len(parts) > 0 { - return parts[0] - } - } - return "friend" -} - -func formatGreeting(template string, g *Guest) string { - r := strings.NewReplacer( - "{first_name}", firstNameFor(g), - "{last_name}", g.LastName, - "{name}", orDefault(g.Name, firstNameFor(g)), - "{email}", g.Email, - ) - return r.Replace(template) -} - -func orDefault(v, def string) string { - if v != "" { - return v - } - return def -} - -func orDefaultInt(v, def int) int { - if v > 0 { - return v - } - return def -} diff --git a/plugins/actions/luma_checkin/luma_checkin_test.go b/plugins/actions/luma_checkin/luma_checkin_test.go deleted file mode 100644 index 20a1f711d8..0000000000 --- a/plugins/actions/luma_checkin/luma_checkin_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package luma_checkin_test - -import ( - "context" - "errors" - "strings" - "sync" - "sync/atomic" - "testing" - - "github.com/openmind/om1/plugins/actions/luma_checkin" -) - -type fakeGuestClient struct { - guest *luma_checkin.Guest - getErr error - updateErr error - getCalls int32 - updateCalls int32 - lastStatusArgs struct { - eventAPIID, guestAPIID, status string - } - mu sync.Mutex -} - -func (f *fakeGuestClient) GetGuest(_ context.Context, _ string) (*luma_checkin.Guest, error) { - atomic.AddInt32(&f.getCalls, 1) - return f.guest, f.getErr -} - -func (f *fakeGuestClient) UpdateGuestStatus(_ context.Context, eventAPIID, guestAPIID, status string) error { - atomic.AddInt32(&f.updateCalls, 1) - f.mu.Lock() - f.lastStatusArgs.eventAPIID = eventAPIID - f.lastStatusArgs.guestAPIID = guestAPIID - f.lastStatusArgs.status = status - f.mu.Unlock() - return f.updateErr -} - -type fakeTTS struct { - mu sync.Mutex - texts []string -} - -func (f *fakeTTS) AddText(t string) { - f.mu.Lock() - defer f.mu.Unlock() - f.texts = append(f.texts, t) -} - -func (f *fakeTTS) lastText() string { - f.mu.Lock() - defer f.mu.Unlock() - if len(f.texts) == 0 { - return "" - } - return f.texts[len(f.texts)-1] -} - -func newTestConnector(client luma_checkin.GuestLookup, cfg luma_checkin.Config) (*luma_checkin.TestConnector, *fakeTTS) { - tts := &fakeTTS{} - c := luma_checkin.NewTestConnector(client, cfg, tts) - return c, tts -} - -func defaultCfg() luma_checkin.Config { - return luma_checkin.Config{ - APIKey: "k", - EventAPIID: "evt-1", - GreetingTemplate: luma_checkin.DefaultGreetingTemplate, - StatusValue: luma_checkin.DefaultStatus, - } -} - -func TestConnectHappyPath(t *testing.T) { - client := &fakeGuestClient{guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-1"}} - c, tts := newTestConnector(client, defaultCfg()) - - out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) - if err != nil { - t.Fatalf("Connect: %v", err) - } - if got, want := out.(string), "checked_in: Ada"; got != want { - t.Errorf("output: got %q want %q", got, want) - } - if atomic.LoadInt32(&client.updateCalls) != 1 { - t.Errorf("expected 1 update call, got %d", client.updateCalls) - } - if client.lastStatusArgs.guestAPIID != "g-1" || client.lastStatusArgs.status != luma_checkin.DefaultStatus { - t.Errorf("update args: %+v", client.lastStatusArgs) - } - if !strings.Contains(tts.lastText(), "Ada") { - t.Errorf("tts text missing first name: %q", tts.lastText()) - } -} - -func TestConnectEventMismatch(t *testing.T) { - client := &fakeGuestClient{guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-other"}} - c, tts := newTestConnector(client, defaultCfg()) - - out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) - if err != nil { - t.Fatalf("Connect: %v", err) - } - if !strings.HasPrefix(out.(string), "checkin_failed:") { - t.Errorf("output: %q", out) - } - if atomic.LoadInt32(&client.updateCalls) != 0 { - t.Errorf("update should not be called on mismatch") - } - if tts.lastText() != "" { - t.Errorf("tts should not fire on mismatch") - } -} - -func TestConnectStatusUpdateFailureNonBlocking(t *testing.T) { - client := &fakeGuestClient{ - guest: &luma_checkin.Guest{APIID: "g-1", FirstName: "Ada", EventAPIID: "evt-1"}, - updateErr: errors.New("boom"), - } - c, tts := newTestConnector(client, defaultCfg()) - - out, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) - if err != nil { - t.Fatalf("Connect: %v", err) - } - s := out.(string) - if !strings.HasPrefix(s, "checked_in: Ada") || !strings.Contains(s, "status update failed") { - t.Errorf("output: %q", s) - } - if !strings.Contains(tts.lastText(), "Ada") { - t.Errorf("tts should still fire when status update fails: %q", tts.lastText()) - } -} - -func TestConnectGuestNotFound(t *testing.T) { - client := &fakeGuestClient{getErr: luma_checkin.ErrLumaNotFound} - c, _ := newTestConnector(client, defaultCfg()) - - out, err := c.Connect(context.Background(), map[string]any{"pk": "g-x"}) - if err != nil { - t.Fatalf("Connect: %v", err) - } - if !strings.HasPrefix(out.(string), "checkin_failed:") { - t.Errorf("output: %q", out) - } -} - -func TestConnectAuthFailure(t *testing.T) { - client := &fakeGuestClient{getErr: luma_checkin.ErrLumaUnauthorized} - c, _ := newTestConnector(client, defaultCfg()) - - _, err := c.Connect(context.Background(), map[string]any{"pk": "g-1"}) - if err == nil { - t.Fatalf("expected error on auth failure") - } -} - -func TestConnectEmptyPk(t *testing.T) { - client := &fakeGuestClient{} - c, _ := newTestConnector(client, defaultCfg()) - - _, err := c.Connect(context.Background(), map[string]any{"pk": ""}) - if err == nil { - t.Fatalf("expected error on empty pk") - } - if atomic.LoadInt32(&client.getCalls) != 0 { - t.Errorf("should not call GetGuest with empty pk") - } -} - -func TestFirstNameFallbacks(t *testing.T) { - cases := []struct { - guest *luma_checkin.Guest - want string - }{ - {&luma_checkin.Guest{FirstName: "Ada"}, "Ada"}, - {&luma_checkin.Guest{Name: "Ada Lovelace"}, "Ada"}, - {&luma_checkin.Guest{}, "friend"}, - } - for _, tc := range cases { - if got := luma_checkin.FirstNameFor(tc.guest); got != tc.want { - t.Errorf("FirstNameFor(%+v): got %q want %q", tc.guest, got, tc.want) - } - } -} - -func TestFormatGreetingTokens(t *testing.T) { - g := &luma_checkin.Guest{FirstName: "Ada", LastName: "Lovelace", Name: "Ada Lovelace", Email: "ada@example.com"} - got := luma_checkin.FormatGreeting("Hi {first_name} {last_name} <{email}>", g) - want := "Hi Ada Lovelace " - if got != want { - t.Errorf("got %q want %q", got, want) - } -} diff --git a/plugins/actions/luma_checkin/luma_client.go b/plugins/actions/luma_checkin/luma_client.go deleted file mode 100644 index 8eacdacc35..0000000000 --- a/plugins/actions/luma_checkin/luma_client.go +++ /dev/null @@ -1,198 +0,0 @@ -package luma_checkin - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/openmind/om1/internal/httpclient" -) - -const ( - defaultLumaBaseURL = "https://public-api.luma.com" - getGuestPath = "/v1/event/get-guest" - updateGuestStatusPath = "/v1/event/update-guest-status" -) - -var ( - errLumaNotFound = errors.New("luma: guest not found") - errLumaUnauthorized = errors.New("luma: unauthorized") -) - -// Guest is a flattened view of the guest record returned by Luma. Field names -// follow the API's snake_case JSON. Unknown fields are ignored. -type Guest struct { - APIID string `json:"api_id"` - Name string `json:"name"` - FirstName string `json:"first_name"` - LastName string `json:"last_name"` - Email string `json:"email"` - EventAPIID string `json:"event_api_id"` - CheckedInAt string `json:"checked_in_at"` - ApprovalStatus string `json:"approval_status"` -} - -// guestEnvelope handles Luma's nested response shape: {"guest": {...}, "event": {...}}. -// Some endpoints return the guest object directly; we try the envelope first. -type guestEnvelope struct { - Guest *Guest `json:"guest"` - Event *struct { - APIID string `json:"api_id"` - } `json:"event"` -} - -// httpDoer is satisfied by *http.Client; abstracted for tests. -type httpDoer interface { - Do(req *http.Request) (*http.Response, error) -} - -type lumaClient struct { - baseURL string - apiKey string - eventAPIID string - http httpDoer - timeout time.Duration -} - -func newLumaClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *lumaClient { - if baseURL == "" { - baseURL = defaultLumaBaseURL - } - if timeout <= 0 { - timeout = 5 * time.Second - } - return &lumaClient{ - baseURL: strings.TrimRight(baseURL, "/"), - apiKey: apiKey, - eventAPIID: eventAPIID, - http: httpclient.Default(), - timeout: timeout, - } -} - -// GetGuest fetches the guest record for the given key (g- or ticket key). -// Luma's get-guest requires both event_id (the configured event) and id (the -// guest/ticket key from the QR). -func (c *lumaClient) GetGuest(ctx context.Context, pk string) (*Guest, error) { - ctx, cancel := context.WithTimeout(ctx, c.timeout) - defer cancel() - - q := url.Values{ - "event_id": []string{c.eventAPIID}, - "id": []string{pk}, - } - endpoint := c.baseURL + getGuestPath + "?" + q.Encode() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, err - } - c.setAuth(req) - - resp, err := c.do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, _ := io.ReadAll(resp.Body) - switch resp.StatusCode { - case http.StatusOK: - // fallthrough to decode - case http.StatusNotFound: - return nil, errLumaNotFound - case http.StatusUnauthorized, http.StatusForbidden: - return nil, errLumaUnauthorized - default: - return nil, fmt.Errorf("luma get-guest %d: %s", resp.StatusCode, string(body)) - } - - var env guestEnvelope - if err := json.Unmarshal(body, &env); err == nil && env.Guest != nil { - g := env.Guest - if g.EventAPIID == "" && env.Event != nil { - g.EventAPIID = env.Event.APIID - } - return g, nil - } - var bare Guest - if err := json.Unmarshal(body, &bare); err != nil { - return nil, fmt.Errorf("luma get-guest decode: %w", err) - } - return &bare, nil -} - -// UpdateGuestStatus flips the guest's approval/check-in status. Best-effort: the -// exact body shape Luma expects is under-documented, so the action treats -// failures here as non-fatal. -func (c *lumaClient) UpdateGuestStatus(ctx context.Context, eventAPIID, guestAPIID, status string) error { - ctx, cancel := context.WithTimeout(ctx, c.timeout) - defer cancel() - - body, _ := json.Marshal(map[string]any{ - "event_id": eventAPIID, - "id": guestAPIID, - "status": status, - }) - endpoint := c.baseURL + updateGuestStatusPath - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - c.setAuth(req) - - resp, err := c.do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return nil - } - rb, _ := io.ReadAll(resp.Body) - return fmt.Errorf("luma update-guest-status %d: %s", resp.StatusCode, string(rb)) -} - -func (c *lumaClient) setAuth(req *http.Request) { - req.Header.Set("x-luma-api-key", c.apiKey) - req.Header.Set("Accept", "application/json") -} - -// do issues req with one retry on 429 after a short backoff. -func (c *lumaClient) do(req *http.Request) (*http.Response, error) { - resp, err := c.http.Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusTooManyRequests { - return resp, nil - } - - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - - t := time.NewTimer(500 * time.Millisecond) - defer t.Stop() - select { - case <-req.Context().Done(): - return nil, req.Context().Err() - case <-t.C: - } - - if req.GetBody != nil { - body, berr := req.GetBody() - if berr != nil { - return nil, berr - } - req.Body = body - } - return c.http.Do(req) -} diff --git a/plugins/actions/luma_checkin/luma_client_test.go b/plugins/actions/luma_checkin/luma_client_test.go deleted file mode 100644 index d828a50ea8..0000000000 --- a/plugins/actions/luma_checkin/luma_client_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package luma_checkin_test - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/openmind/om1/plugins/actions/luma_checkin" -) - -func newTestClient(t *testing.T, handler http.HandlerFunc) (*luma_checkin.LumaClient, *httptest.Server) { - t.Helper() - srv := httptest.NewServer(handler) - c := luma_checkin.NewLumaClient(srv.URL, "test-key", "evt-test", 2*time.Second) - return c, srv -} - -func TestGetGuestEnvelopeOK(t *testing.T) { - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != luma_checkin.GetGuestPath { - t.Errorf("path: got %q want %q", r.URL.Path, luma_checkin.GetGuestPath) - } - if r.URL.Query().Get("id") != "g-123" { - t.Errorf("id query: got %q want g-123", r.URL.Query().Get("id")) - } - if r.URL.Query().Get("event_id") != "evt-test" { - t.Errorf("event_id query: got %q want evt-test", r.URL.Query().Get("event_id")) - } - if r.Header.Get("x-luma-api-key") != "test-key" { - t.Errorf("missing api key header: %q", r.Header.Get("x-luma-api-key")) - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"guest":{"api_id":"g-123","first_name":"Ada","name":"Ada Lovelace","email":"ada@example.com"},"event":{"api_id":"evt-abc"}}`) - }) - defer srv.Close() - - g, err := c.GetGuest(context.Background(), "g-123") - if err != nil { - t.Fatalf("GetGuest: %v", err) - } - if g.APIID != "g-123" || g.FirstName != "Ada" || g.EventAPIID != "evt-abc" { - t.Errorf("unexpected guest: %+v", g) - } -} - -func TestGetGuestBareOK(t *testing.T) { - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, `{"api_id":"g-9","first_name":"Bo","event_api_id":"evt-1"}`) - }) - defer srv.Close() - - g, err := c.GetGuest(context.Background(), "g-9") - if err != nil { - t.Fatalf("GetGuest: %v", err) - } - if g.FirstName != "Bo" || g.EventAPIID != "evt-1" { - t.Errorf("unexpected guest: %+v", g) - } -} - -func TestGetGuestNotFound(t *testing.T) { - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - }) - defer srv.Close() - - _, err := c.GetGuest(context.Background(), "g-x") - if !errors.Is(err, luma_checkin.ErrLumaNotFound) { - t.Fatalf("want ErrLumaNotFound, got %v", err) - } -} - -func TestGetGuestUnauthorized(t *testing.T) { - for _, code := range []int{http.StatusUnauthorized, http.StatusForbidden} { - code := code - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(code) - }) - _, err := c.GetGuest(context.Background(), "g-x") - srv.Close() - if !errors.Is(err, luma_checkin.ErrLumaUnauthorized) { - t.Errorf("status %d: want ErrLumaUnauthorized, got %v", code, err) - } - } -} - -func TestGetGuestRetriesOn429(t *testing.T) { - var calls int - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - calls++ - if calls == 1 { - w.WriteHeader(http.StatusTooManyRequests) - return - } - _, _ = io.WriteString(w, `{"api_id":"g-1","first_name":"A","event_api_id":"e"}`) - }) - defer srv.Close() - - g, err := c.GetGuest(context.Background(), "g-1") - if err != nil { - t.Fatalf("GetGuest: %v", err) - } - if calls != 2 { - t.Errorf("expected 2 attempts, got %d", calls) - } - if g.FirstName != "A" { - t.Errorf("unexpected guest: %+v", g) - } -} - -func TestUpdateGuestStatusOK(t *testing.T) { - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("method: got %s want POST", r.Method) - } - if r.URL.Path != luma_checkin.UpdateGuestStatusPath { - t.Errorf("path: got %q want %q", r.URL.Path, luma_checkin.UpdateGuestStatusPath) - } - if r.Header.Get("Content-Type") != "application/json" { - t.Errorf("content-type: %q", r.Header.Get("Content-Type")) - } - var body map[string]string - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - t.Fatalf("decode body: %v", err) - } - if body["event_id"] != "evt-1" || body["id"] != "g-1" || body["status"] != "checked_in" { - t.Errorf("body: %+v", body) - } - w.WriteHeader(http.StatusOK) - }) - defer srv.Close() - - if err := c.UpdateGuestStatus(context.Background(), "evt-1", "g-1", "checked_in"); err != nil { - t.Fatalf("UpdateGuestStatus: %v", err) - } -} - -func TestUpdateGuestStatusErrorBubbles(t *testing.T) { - c, srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - _, _ = io.WriteString(w, `{"error":"bad shape"}`) - }) - defer srv.Close() - - err := c.UpdateGuestStatus(context.Background(), "evt-1", "g-1", "checked_in") - if err == nil || !strings.Contains(err.Error(), "400") { - t.Fatalf("want 400 error, got %v", err) - } -} From 85c797afcb339709fa34a4cf93585fbfd47f468c Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 16:59:36 -0700 Subject: [PATCH 09/40] removed stale entry from actions.go --- plugins/actions/actions.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/actions/actions.go b/plugins/actions/actions.go index 4809e14d75..ef887be56f 100644 --- a/plugins/actions/actions.go +++ b/plugins/actions/actions.go @@ -3,7 +3,6 @@ package actions import ( _ "github.com/openmind/om1/plugins/actions/emotion" _ "github.com/openmind/om1/plugins/actions/greeting_conversation" - _ "github.com/openmind/om1/plugins/actions/luma_checkin" _ "github.com/openmind/om1/plugins/actions/navigation" _ "github.com/openmind/om1/plugins/actions/speak" _ "github.com/openmind/om1/plugins/actions/unitree/g1/arm" From 1367ca70caaa812185b76d6daa876eb80a3939ea Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 17:00:19 -0700 Subject: [PATCH 10/40] updated local config for luma plugin --- config/luma_checkin_local.json5 | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 index adae474aed..65bccdba85 100644 --- a/config/luma_checkin_local.json5 +++ b/config/luma_checkin_local.json5 @@ -1,21 +1,27 @@ { -version: "v1.0.0", + version: "v1.0.5", hertz: 0.2, name: "luma_checkin_local", api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a test harness for the Luma check-in flow. The Vision input emits a line of the form 'qr_scan: pk= event=' when it sees a Luma check-in QR code, or 'none' when it does not. When you see 'qr_scan: pk=...', immediately call the luma_checkin action with that pk value. When you see 'none', do nothing. Do not narrate.", - system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", + system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", + system_governance: "Only call speak. Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes.", agent_inputs: [ { type: "QRScanner", config: { camera_index: 0, - capture_fps: 15, + capture_fps: 30, decode_fps: 5, - resolution_width: 640, - resolution_height: 480, + resolution_width: 1280, + resolution_height: 720, jpeg_quality: 60, dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + request_timeout_seconds: 4, + }, }, }, ], @@ -26,21 +32,15 @@ version: "v1.0.0", history_length: 4, }, }, + action_execution_mode: "concurrent", agent_actions: [ { - name: "luma_checkin", - llm_label: "luma_checkin", - connector: "api", + name: "speak", + llm_label: "speak", + implementation: "passthrough", + connector: "elevenlabs_tts", config: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Confirmed for {email}.", - request_timeout_seconds: 5, - tts: { - provider: "elevenlabs", - elevenlabs_api_key: "${ELEVENLABS_API_KEY}", - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, + voice_id: "PoHUWWWMHFrA8z7Q88pu", }, }, ], From 82ce2208716abeff05192b6879598fd0af2bdaf2 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 17:01:58 -0700 Subject: [PATCH 11/40] updated configs --- config/luma_checkin.json5 | 45 ++++++++++++----------------------- config/luma_checkin_go2.json5 | 34 +++++++++++++------------- 2 files changed, 32 insertions(+), 47 deletions(-) diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 index d73c98042a..5d93be00ae 100644 --- a/config/luma_checkin.json5 +++ b/config/luma_checkin.json5 @@ -1,25 +1,10 @@ { - // Luma event check-in: scans QR codes via camera, looks guests up via Luma's - // external check-in API, speaks a personalized welcome, and (via the existing - // emotion + unitree_g1_arm plugins) shows a happy face and waves. - // - // Required env vars: - // LUMA_API_KEY - Luma API key with external-check-in scope - // LUMA_EVENT_ID - api_id of the event to check guests into (e.g. evt-...) - // ELEVENLABS_API_KEY - ElevenLabs TTS API key - version: "v1.0.0", + version: "v1.0.5", hertz: 0.2, name: "luma_checkin", api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot stationed at a Luma event check-in desk.\ -\n\ -When you receive a 'qr_scan: pk=...' input from the QR scanner, respond in a single turn with three actions, all called concurrently:\n\ - 1. luma_checkin with that pk (this looks up the guest and speaks the welcome).\n\ - 2. emotion with action 'happy'.\n\ - 3. unitree_g1_arm with action 'face_wave'.\n\ -\n\ -After luma_checkin returns, you may optionally make one short follow-up comment. Treat each scan as exactly one check-in (the input layer already debounces).", - system_governance: "Never check the same guest in twice within the same scan window. Never speak before luma_checkin returns.", + system_prompt_base: "You are a friendly event greeter robot stationed at a Luma event check-in desk. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure (guest not registered, auth issue, etc).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': respond in a single turn with these actions, all called concurrently:\n 1. speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names.\n 2. emotion with action 'happy'.\n 3. unitree_g1_arm with action 'face_wave'.\n\nOn 'qr_scan_failed:': do NOT call speak; instead call emotion with 'sad'. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. The input layer debounces, so each scan should be greeted exactly once.", + system_governance: "Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes. Never check the same guest in twice within the same scan window.", agent_inputs: [ { type: "QRScanner", @@ -31,6 +16,12 @@ After luma_checkin returns, you may optionally make one short follow-up comment. resolution_height: 480, jpeg_quality: 60, dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Glad you made it.", + request_timeout_seconds: 4, + }, }, }, ], @@ -44,19 +35,13 @@ After luma_checkin returns, you may optionally make one short follow-up comment. action_execution_mode: "concurrent", agent_actions: [ { - name: "luma_checkin", - llm_label: "luma_checkin", - connector: "api", + name: "speak", + llm_label: "speak", + implementation: "passthrough", + connector: "elevenlabs_tts", config: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Glad you made it.", - request_timeout_seconds: 5, - tts: { - provider: "elevenlabs", - elevenlabs_api_key: "${ELEVENLABS_API_KEY}", - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, + api_key: "${OM_API_KEY:-openmind_free}", + voice_id: "PoHUWWWMHFrA8z7Q88pu", }, }, { diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 index 4c72be9fe7..899698dc50 100644 --- a/config/luma_checkin_go2.json5 +++ b/config/luma_checkin_go2.json5 @@ -1,18 +1,23 @@ { -version: "v1.0.0", + version: "v1.0.5", hertz: 0.2, - name: "luma_checkin_local", + name: "luma_checkin_go2", api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a test harness for the Luma check-in flow. The Vision input emits a line of the form 'qr_scan: pk= event=' when it sees a Luma check-in QR code, or 'none' when it does not. When you see 'qr_scan: pk=...', immediately call the luma_checkin action with that pk value. When you see 'none', do nothing. Do not narrate.", - system_governance: "Only call luma_checkin. Never invent pk values; only use the one in the latest qr_scan input. Ignore 'none' inputs.", + system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", + system_governance: "Only call speak. Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes.", agent_inputs: [ { type: "QRScannerRTSP", config: { - camera_index: 0, capture_fps: 15, decode_fps: 5, dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + request_timeout_seconds: 4, + }, }, }, ], @@ -23,21 +28,16 @@ version: "v1.0.0", history_length: 4, }, }, + action_execution_mode: "concurrent", agent_actions: [ { - name: "luma_checkin", - llm_label: "luma_checkin", - connector: "api", + name: "speak", + llm_label: "speak", + implementation: "passthrough", + connector: "elevenlabs_tts", config: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Confirmed for {email}.", - request_timeout_seconds: 5, - tts: { - provider: "elevenlabs", - elevenlabs_api_key: "${ELEVENLABS_API_KEY}", - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, + api_key: "${OM_API_KEY:-openmind_free}", + voice_id: "PoHUWWWMHFrA8z7Q88pu", }, }, ], From 3df74501371b4f557340c0abe5e6eb61285690df Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 17:25:52 -0700 Subject: [PATCH 12/40] update config --- config/luma_checkin_go2.json5 | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 index 899698dc50..0936970b0f 100644 --- a/config/luma_checkin_go2.json5 +++ b/config/luma_checkin_go2.json5 @@ -9,8 +9,6 @@ { type: "QRScannerRTSP", config: { - capture_fps: 15, - decode_fps: 5, dedupe_window_seconds: 30, luma: { api_key: "${LUMA_API_KEY}", From 0490d0b7b5149e68196d99dc641d2dc0e422d3b1 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 18:28:52 -0700 Subject: [PATCH 13/40] fix greeting issues --- config/luma_checkin_go2.json5 | 7 +- config/luma_checkin_local.json5 | 6 +- plugins/inputs/qr_scanner/export_test.go | 49 ++++- .../qr_scanner/format_scan_message_test.go | 172 ++++++++++++++++++ plugins/inputs/qr_scanner/qr_scanner.go | 82 +++++++-- 5 files changed, 295 insertions(+), 21 deletions(-) create mode 100644 plugins/inputs/qr_scanner/format_scan_message_test.go diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 index 0936970b0f..04947b767e 100644 --- a/config/luma_checkin_go2.json5 +++ b/config/luma_checkin_go2.json5 @@ -4,7 +4,7 @@ name: "luma_checkin_go2", api_key: "${OM_API_KEY:-openmind_free}", system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", - system_governance: "Only call speak. Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes.", + system_governance: "", agent_inputs: [ { type: "QRScannerRTSP", @@ -13,7 +13,7 @@ luma: { api_key: "${LUMA_API_KEY}", event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", request_timeout_seconds: 4, }, }, @@ -23,7 +23,7 @@ type: "GeminiLLM", config: { agent_name: "Greeter", - history_length: 4, + history_length: 0, }, }, action_execution_mode: "concurrent", @@ -34,7 +34,6 @@ implementation: "passthrough", connector: "elevenlabs_tts", config: { - api_key: "${OM_API_KEY:-openmind_free}", voice_id: "PoHUWWWMHFrA8z7Q88pu", }, }, diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 index 65bccdba85..b56a011735 100644 --- a/config/luma_checkin_local.json5 +++ b/config/luma_checkin_local.json5 @@ -4,7 +4,7 @@ name: "luma_checkin_local", api_key: "${OM_API_KEY:-openmind_free}", system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", - system_governance: "Only call speak. Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes.", + system_governance: "", agent_inputs: [ { type: "QRScanner", @@ -19,7 +19,7 @@ luma: { api_key: "${LUMA_API_KEY}", event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Confirmed for {email}.", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", request_timeout_seconds: 4, }, }, @@ -29,7 +29,7 @@ type: "GeminiLLM", config: { agent_name: "Greeter", - history_length: 4, + history_length: 0, }, }, action_execution_mode: "concurrent", diff --git a/plugins/inputs/qr_scanner/export_test.go b/plugins/inputs/qr_scanner/export_test.go index c3f69458aa..827ef8aa93 100644 --- a/plugins/inputs/qr_scanner/export_test.go +++ b/plugins/inputs/qr_scanner/export_test.go @@ -1,6 +1,13 @@ package qr_scanner -import "time" +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/openmind/om1/internal/providers/luma" +) var ( ParseLumaCheckinURL = parseLumaCheckinURL @@ -14,3 +21,43 @@ func NewDebouncer(window time.Duration) *Debouncer { return newDebouncer(window) func (d *Debouncer) SetNow(f func() time.Time) { d.now = f } func (d *Debouncer) Has(key string) bool { _, ok := d.seen[key]; return ok } + +// GuestLookup is exported for tests so they can inject a fake without touching +// the http stack. +type GuestLookup = guestLookup + +// NewTestSensor builds a sensor wired with the given lookup, greeting template +// and expected event id. The frame source and debouncer are not used by tests +// that only exercise formatScanMessage. +func NewTestSensor(lookup GuestLookup, greetingTmpl, expectedEventID string, timeout time.Duration) *TestSensor { + return &TestSensor{&sensor{ + log: zap.NewNop(), + luma: lookup, + greetingTmpl: greetingTmpl, + lumaTimeout: timeout, + expectedEventID: expectedEventID, + }} +} + +type TestSensor struct{ *sensor } + +func (s *TestSensor) FormatScanMessage(ctx context.Context, pk, eventID string) string { + return s.sensor.formatScanMessage(ctx, pk, eventID) +} + +// SetSpeaker injects a fake ttsSpeaker so tests can verify the direct-TTS +// path without touching the real ElevenLabs singleton. +func (s *TestSensor) SetSpeaker(sp Speaker) { s.sensor.speak = sp } + +// Speaker is the test-side alias for the unexported ttsSpeaker interface. +type Speaker = ttsSpeaker + +// FakeLookup implements GuestLookup for tests. +type FakeLookup struct { + Guest *luma.Guest + Err error +} + +func (f *FakeLookup) GetGuest(_ context.Context, _ string) (*luma.Guest, error) { + return f.Guest, f.Err +} diff --git a/plugins/inputs/qr_scanner/format_scan_message_test.go b/plugins/inputs/qr_scanner/format_scan_message_test.go new file mode 100644 index 0000000000..7bcef4bd49 --- /dev/null +++ b/plugins/inputs/qr_scanner/format_scan_message_test.go @@ -0,0 +1,172 @@ +package qr_scanner_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/openmind/om1/internal/providers/luma" + "github.com/openmind/om1/plugins/inputs/qr_scanner" +) + +func TestFormatScanMessage_NoLumaConfigured(t *testing.T) { + s := qr_scanner.NewTestSensor(nil, "", "", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := "qr_scan: pk=g-1 event=evt-abc" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_EventMismatch(t *testing.T) { + lookup := &qr_scanner.FakeLookup{} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-expected", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-other") + want := "qr_scan_failed: pk=g-1 reason=event_mismatch" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_NotFound(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Err: luma.ErrNotFound} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := "qr_scan_failed: pk=g-1 reason=guest_not_registered" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_Unauthorized(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Err: luma.ErrUnauthorized} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := "qr_scan_failed: pk=g-1 reason=luma_auth" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_GenericError(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Err: errors.New("network down")} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := "qr_scan_failed: pk=g-1 reason=lookup_error" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_EmptyResponse(t *testing.T) { + lookup := &qr_scanner.FakeLookup{} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := "qr_scan_failed: pk=g-1 reason=empty_response" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_Success(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ + UserFirstName: "Prachi", + UserLastName: "Singh", + UserEmail: "prachi@example.com", + }} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}! Confirmed for {email}.", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := `qr_scan: name=Prachi greeting="Welcome, Prachi! Confirmed for prachi@example.com."` + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_SuccessStripsQuotesInGreeting(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ + UserFirstName: "Ada", + UserName: `Ada "the original" Lovelace`, + }} + s := qr_scanner.NewTestSensor(lookup, `Welcome, {name}!`, "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + want := `qr_scan: name=Ada greeting="Welcome, Ada 'the original' Lovelace!"` + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatScanMessage_EmptyEventIDSkipsMismatch(t *testing.T) { + // When the QR doesn't carry an event id, we trust the configured expected one + // and proceed with lookup rather than rejecting. + lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{UserFirstName: "Sam"}} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + got := s.FormatScanMessage(context.Background(), "g-1", "") + want := `qr_scan: name=Sam greeting="Welcome, Sam!"` + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +// fakeSpeaker captures AddText calls so tests can assert the direct-TTS path. +type fakeSpeaker struct{ texts []string } + +func (f *fakeSpeaker) AddText(t string) { f.texts = append(f.texts, t) } + +func TestFormatScanMessage_DirectTTSOnSuccess(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ + UserFirstName: "Prachi", + UserEmail: "prachi@example.com", + }} + speaker := &fakeSpeaker{} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}! Confirmed for {email}.", "evt-abc", time.Second) + s.SetSpeaker(speaker) + + _ = s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + + if len(speaker.texts) != 1 { + t.Fatalf("expected 1 AddText call, got %d", len(speaker.texts)) + } + want := "Welcome, Prachi! Confirmed for prachi@example.com." + if speaker.texts[0] != want { + t.Fatalf("got %q want %q", speaker.texts[0], want) + } +} + +func TestFormatScanMessage_NoSpeakOnFailure(t *testing.T) { + cases := []struct { + name string + err error + }{ + {"not_found", luma.ErrNotFound}, + {"unauthorized", luma.ErrUnauthorized}, + {"generic", errors.New("boom")}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Err: tc.err} + speaker := &fakeSpeaker{} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + s.SetSpeaker(speaker) + + _ = s.FormatScanMessage(context.Background(), "g-1", "evt-abc") + + if len(speaker.texts) != 0 { + t.Fatalf("expected no AddText calls on failure, got %v", speaker.texts) + } + }) + } +} + +func TestFormatScanMessage_NoSpeakOnEventMismatch(t *testing.T) { + lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{UserFirstName: "Sam"}} + speaker := &fakeSpeaker{} + s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) + s.SetSpeaker(speaker) + + _ = s.FormatScanMessage(context.Background(), "g-1", "evt-other") + + if len(speaker.texts) != 0 { + t.Fatalf("expected no AddText on event mismatch, got %v", speaker.texts) + } +} diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go index 7a7cc61a58..c8929cacea 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -15,6 +15,7 @@ import ( "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" "github.com/openmind/om1/internal/providers/luma" + "github.com/openmind/om1/internal/providers/tts" video "github.com/openmind/om1/internal/providers/vlm" ) @@ -46,17 +47,31 @@ type LumaConfig struct { RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` } +// SpeakConfig, when set, makes the scanner push the resolved greeting straight +// to the ElevenLabs TTS provider on a successful Luma lookup, bypassing the +// LLM entirely. This avoids cortex-tick hallucinations where Gemini invents +// greetings on empty observations. +type SpeakConfig struct { + APIKey string `json:"api_key"` + ElevenLabsAPIKey string `json:"elevenlabs_api_key"` + VoiceID string `json:"voice_id"` + ModelID string `json:"model_id"` + OutputFormat string `json:"output_format"` + Rate int `json:"rate"` +} + // Config holds the JSON configuration for the QRScanner input plugin. type Config struct { - CameraIndex int `json:"camera_index"` - RTSPURL string `json:"rtsp_url"` - CaptureFPS int `json:"capture_fps"` - DecodeFPS int `json:"decode_fps"` - Width int `json:"resolution_width"` - Height int `json:"resolution_height"` - JPEGQuality int `json:"jpeg_quality"` - DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` - Luma *LumaConfig `json:"luma"` + CameraIndex int `json:"camera_index"` + RTSPURL string `json:"rtsp_url"` + CaptureFPS int `json:"capture_fps"` + DecodeFPS int `json:"decode_fps"` + Width int `json:"resolution_width"` + Height int `json:"resolution_height"` + JPEGQuality int `json:"jpeg_quality"` + DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` + Luma *LumaConfig `json:"luma"` + Speak *SpeakConfig `json:"speak"` } type frameSource interface { @@ -70,6 +85,12 @@ type guestLookup interface { GetGuest(ctx context.Context, pk string) (*luma.Guest, error) } +// ttsSpeaker is the slice of *tts.ElevenLabsProvider the scanner uses to +// bypass the LLM and push a greeting straight to audio. Abstracted for tests. +type ttsSpeaker interface { + AddText(text string) +} + type sensor struct { name string cfg Config @@ -77,10 +98,11 @@ type sensor struct { source frameSource debouncer *debouncer - luma guestLookup - greetingTmpl string - lumaTimeout time.Duration - expectedEventID string + luma guestLookup + greetingTmpl string + lumaTimeout time.Duration + expectedEventID string + speak ttsSpeaker mu sync.Mutex messages []inputs.Message @@ -163,9 +185,39 @@ func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { ) } + if cfg.Speak != nil && cfg.Speak.APIKey != "" { + s.speak = newTTSSpeaker(*cfg.Speak, log) + log.Info("direct tts enabled (greeting bypasses LLM)") + } + return s } +// newTTSSpeaker builds the singleton ElevenLabs provider with the supplied +// config. Wrapped so tests can substitute via setSpeak. +func newTTSSpeaker(cfg SpeakConfig, log *zap.Logger) ttsSpeaker { + if cfg.VoiceID == "" { + cfg.VoiceID = tts.DefaultVoiceID + } + if cfg.ModelID == "" { + cfg.ModelID = tts.DefaultModelID + } + if cfg.OutputFormat == "" { + cfg.OutputFormat = tts.DefaultOutputFormat + } + if cfg.Rate == 0 { + cfg.Rate = tts.DefaultRate + } + return tts.ElevenLabs(tts.ElevenLabsConfig{ + APIKey: cfg.APIKey, + ElevenLabsAPIKey: cfg.ElevenLabsAPIKey, + VoiceID: cfg.VoiceID, + ModelID: cfg.ModelID, + OutputFormat: cfg.OutputFormat, + Rate: cfg.Rate, + }, log.Named("speak/elevenlabs_tts")) +} + func parseConfig(configMap map[string]any) Config { var cfg Config if b, err := json.Marshal(configMap); err == nil { @@ -366,6 +418,10 @@ func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) stri name := luma.FirstName(guest) greeting := luma.FormatGreeting(s.greetingTmpl, guest) + if s.speak != nil { + s.speak.AddText(greeting) + s.log.Info("greeting pushed to tts", zap.String("pk", pk), zap.String("name", name)) + } greeting = strings.ReplaceAll(greeting, `"`, `'`) s.log.Info("luma lookup ok", zap.String("pk", pk), zap.String("name", name)) return fmt.Sprintf(`qr_scan: name=%s greeting="%s"`, name, greeting) From 89ae062bb8db378f2395491a821301cfa2c3b4a1 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 18:32:13 -0700 Subject: [PATCH 14/40] fix greeting config --- config/luma_checkin_go2.json5 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 index 04947b767e..d4e1f483d7 100644 --- a/config/luma_checkin_go2.json5 +++ b/config/luma_checkin_go2.json5 @@ -1,7 +1,7 @@ { version: "v1.0.5", hertz: 0.2, - name: "luma_checkin_go2", + name: "luma_checkin_local", api_key: "${OM_API_KEY:-openmind_free}", system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", system_governance: "", @@ -9,6 +9,9 @@ { type: "QRScannerRTSP", config: { + camera_index: 0, + capture_fps: 30, + decode_fps: 5, dedupe_window_seconds: 30, luma: { api_key: "${LUMA_API_KEY}", From 9c22ea6e7c97f4b639f4f069391247abb0824821 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 18:37:31 -0700 Subject: [PATCH 15/40] fix config --- config/luma_checkin_go2.json5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 index d4e1f483d7..1e5e08c983 100644 --- a/config/luma_checkin_go2.json5 +++ b/config/luma_checkin_go2.json5 @@ -1,7 +1,7 @@ { version: "v1.0.5", hertz: 0.2, - name: "luma_checkin_local", + name: "luma_checkin_go2", api_key: "${OM_API_KEY:-openmind_free}", system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", system_governance: "", From 5b3ea8080f6b3ae836716f94a006692f542db064 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 18:56:54 -0700 Subject: [PATCH 16/40] update configs --- config/luma_checkin.json5 | 23 ++++++------------ config/luma_checkin_go2.json5 | 44 ----------------------------------- 2 files changed, 7 insertions(+), 60 deletions(-) delete mode 100644 config/luma_checkin_go2.json5 diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 index 5d93be00ae..13b3e5990e 100644 --- a/config/luma_checkin.json5 +++ b/config/luma_checkin.json5 @@ -1,25 +1,22 @@ { version: "v1.0.5", hertz: 0.2, - name: "luma_checkin", + name: "luma_checkin_go2", api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot stationed at a Luma event check-in desk. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure (guest not registered, auth issue, etc).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': respond in a single turn with these actions, all called concurrently:\n 1. speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names.\n 2. emotion with action 'happy'.\n 3. unitree_g1_arm with action 'face_wave'.\n\nOn 'qr_scan_failed:': do NOT call speak; instead call emotion with 'sad'. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. The input layer debounces, so each scan should be greeted exactly once.", - system_governance: "Never invent greeting text; only repeat the greeting string already present in the observation, exactly as written between the double quotes. Never check the same guest in twice within the same scan window.", + system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", + system_governance: "", agent_inputs: [ { - type: "QRScanner", + type: "QRScannerRTSP", config: { camera_index: 0, - capture_fps: 15, + capture_fps: 30, decode_fps: 5, - resolution_width: 640, - resolution_height: 480, - jpeg_quality: 60, dedupe_window_seconds: 30, luma: { api_key: "${LUMA_API_KEY}", event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! Glad you made it.", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", request_timeout_seconds: 4, }, }, @@ -29,7 +26,7 @@ type: "GeminiLLM", config: { agent_name: "Greeter", - history_length: 6, + history_length: 0, }, }, action_execution_mode: "concurrent", @@ -40,7 +37,6 @@ implementation: "passthrough", connector: "elevenlabs_tts", config: { - api_key: "${OM_API_KEY:-openmind_free}", voice_id: "PoHUWWWMHFrA8z7Q88pu", }, }, @@ -49,10 +45,5 @@ llm_label: "emotion", connector: "zenoh", }, - { - name: "unitree_g1_arm", - llm_label: "unitree_g1_arm", - connector: "zenoh", - }, ], } diff --git a/config/luma_checkin_go2.json5 b/config/luma_checkin_go2.json5 deleted file mode 100644 index 1e5e08c983..0000000000 --- a/config/luma_checkin_go2.json5 +++ /dev/null @@ -1,44 +0,0 @@ -{ - version: "v1.0.5", - hertz: 0.2, - name: "luma_checkin_go2", - api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", - system_governance: "", - agent_inputs: [ - { - type: "QRScannerRTSP", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - implementation: "passthrough", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, - }, - ], -} From 7d028b07458a43a6d2e2a1b9a4195006eced1260 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Thu, 11 Jun 2026 19:03:46 -0700 Subject: [PATCH 17/40] fix issues in qr scanner --- internal/providers/luma/client.go | 5 ++++- internal/providers/vlm/video_stream.go | 10 +++++++--- plugins/inputs/qr_scanner/qr_scanner.go | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go index 7671a4d5fc..05b51a07e8 100644 --- a/internal/providers/luma/client.go +++ b/internal/providers/luma/client.go @@ -110,7 +110,10 @@ func (c *Client) GetGuest(ctx context.Context, pk string) (*Guest, error) { } defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("luma get-guest read body: %w", err) + } switch resp.StatusCode { case http.StatusOK: case http.StatusNotFound: diff --git a/internal/providers/vlm/video_stream.go b/internal/providers/vlm/video_stream.go index fb87be6e08..234e0ba6cc 100644 --- a/internal/providers/vlm/video_stream.go +++ b/internal/providers/vlm/video_stream.go @@ -122,10 +122,14 @@ func (v *VideoStream) stream(ctx context.Context, cam string) error { } func (v *VideoStream) logFFmpegStderr(r io.ReadCloser) { + defer r.Close() scanner := bufio.NewScanner(r) for scanner.Scan() { v.log.Warn("VideoStream: ffmpeg", zap.String("stderr", scanner.Text())) } + if err := scanner.Err(); err != nil { + v.log.Warn("VideoStream: ffmpeg stderr read error", zap.Error(err)) + } } // cameraInput returns the ffmpeg input specifier for the configured camera device. @@ -169,11 +173,11 @@ func (v *VideoStream) ffmpegArgs(cam string) []string { "-loglevel", "error", "-f", "v4l2", } + if v.cfg.FPS > 0 { + args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) + } if v.cfg.Width > 0 && v.cfg.Height > 0 { args = append(args, "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height)) - if v.cfg.FPS > 0 { - args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) - } } return append(args, "-i", cam, diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go index c8929cacea..b0de78fc53 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -282,6 +282,9 @@ func (s *sensor) Listen(ctx context.Context) (<-chan any, error) { text, err := decodeQR(frame.JPEG) if err != nil { + if !errors.Is(err, errQRNotFound) { + s.log.Debug("decode error", zap.Error(err)) + } continue } eventID, pk, valid := parseLumaCheckinURL(text) From b788cc2bcda85374bff61bbbec934cb5db7ceb09 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 10:48:05 -0700 Subject: [PATCH 18/40] add default width and height for camera back --- internal/providers/vlm/video_stream.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/providers/vlm/video_stream.go b/internal/providers/vlm/video_stream.go index 234e0ba6cc..3ce9b31990 100644 --- a/internal/providers/vlm/video_stream.go +++ b/internal/providers/vlm/video_stream.go @@ -17,6 +17,8 @@ import ( ) const ( + defaultWidth = 640 + defaultHeight = 480 defaultFPS = 30 defaultJPEGQuality = 30 cameraRetryDelay = 2 * time.Second @@ -40,6 +42,18 @@ type VideoStream struct { } func NewVideoStream(cfg VideoStreamConfig) *VideoStream { + if cfg.FPS <= 0 { + cfg.FPS = defaultFPS + } + + if cfg.Width <= 0 { + cfg.Width = defaultWidth + } + + if cfg.Height <= 0 { + cfg.Height = defaultHeight + } + if cfg.JPEGQuality <= 0 { cfg.JPEGQuality = defaultJPEGQuality } From 8b37a947fa6952c49cc0d999bb46c982fc407555 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 14:12:19 -0700 Subject: [PATCH 19/40] add face size check background plugin --- config/luma_checkin.json5 | 201 ++++++++++++++++++++---- plugins/backgrounds/checkin_complete.go | 139 ++++++++++++++++ plugins/backgrounds/face_size_watch.go | 99 ++++++++++++ 3 files changed, 408 insertions(+), 31 deletions(-) create mode 100644 plugins/backgrounds/checkin_complete.go create mode 100644 plugins/backgrounds/face_size_watch.go diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 index 13b3e5990e..2e9bb9c96b 100644 --- a/config/luma_checkin.json5 +++ b/config/luma_checkin.json5 @@ -1,27 +1,12 @@ { version: "v1.0.5", - hertz: 0.2, name: "luma_checkin_go2", + default_mode: "face_watch", + allow_manual_switching: true, + api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", system_governance: "", - agent_inputs: [ - { - type: "QRScannerRTSP", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], + cortex_llm: { type: "GeminiLLM", config: { @@ -29,21 +14,175 @@ history_length: 0, }, }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - implementation: "passthrough", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", + + modes: { + // Mode 1: Passive face-size watch. Waits until a face bounding box area + // exceeds min_face_area pixels, then transitions to checkin mode. + face_watch: { + display_name: "Face Watch Mode", + description: "Monitors camera for a face close enough to check in. Transitions to checkin once a large-enough face is detected.", + system_prompt_base: "", + hertz: 0.5, + agent_inputs: [], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 0, + }, }, + action_execution_mode: "concurrent", + agent_actions: [], + // FaceSizeWatch polls the face HTTP service and publishes + // face_close_enough: true to ModeContext when a bounding box + // area exceeds min_face_area pixels. + backgrounds: [ + { + type: "FaceSizeWatch", + config: { + face_http_base_url: "http://127.0.0.1:6793", + face_recent_sec: 1.0, + face_poll_interval_sec: 0.5, + min_face_area: 15000, + }, + }, + ], + lifecycle_hooks: [], + }, + + // Mode 2: Active check-in. Announces instructions on entry, runs the QR + // scanner in the background, and lets the LLM speak the personalized + // greeting when a scan arrives. The QR scanner triggers an immediate tick + // (bypassing the hertz timer) on every fresh scan so the LLM responds + // without waiting for the next scheduled cycle. + checkin: { + display_name: "Check-in Mode", + description: "Greets guests by QR code. Announces instructions on entry and speaks the greeting returned by the Luma lookup.", + system_prompt_base: "You are a friendly event check-in robot.\n\ +\n\ +Check-in flow:\n\ +1. First ask the guest: 'Hi there! Are you registered on Luma for this event?'\n\ +2. If they say YES (or similar): say 'Great! Please hold up your QR code to the camera and I will check you in.'\n\ + Then wait for the QR Scanner to pick it up.\n\ +3. If they say NO (or not registered, or similar): say 'No problem! Could you tell me your name so I can check you in?'\n\ + When they give their name, welcome them warmly by name.\n\ +\n\ +QR Scanner handling:\n\ +The QR Scanner input emits one of these messages when it sees a check-in QR:\n\ + - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n\ + - 'qr_scan_failed: pk= reason=' on a failure.\n\ +\n\ +On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text — do not paraphrase or invent names.\n\ +On 'qr_scan_failed:': apologize and ask them to try again or tell you their name instead.\n\ +If there is no qr_scan line in the current observations, continue the conversation naturally.\n\ +Do not invent scan results. Do not narrate.", + hertz: 0.5, + agent_inputs: [ + { + type: "${ASR_INPUT_PLUGIN:-GoogleASRInput}", + config: { + api_version: "v2", + enable_tts_interrupt: false, + rate: 16000, + chunk: 1600, + }, + }, + { + type: "ConversationHistoryInput", + config: { + max_rounds: 5, + }, + }, + { + type: "QRScannerRTSP", + config: { + camera_index: 0, + capture_fps: 30, + decode_fps: 5, + dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", + request_timeout_seconds: 4, + }, + }, + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 5, + }, + }, + action_execution_mode: "concurrent", + agent_actions: [ + { + name: "speak", + llm_label: "speak", + connector: "elevenlabs_tts", + config: { + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + { + name: "emotion", + llm_label: "emotion", + connector: "zenoh", + }, + ], + backgrounds: [ + { + type: "CheckinComplete", + config: { + face_http_base_url: "http://127.0.0.1:6793", + min_face_area: 15000, + poll_interval_sec: 1.0, + scan_io_key: "QRScannerRTSP", + // Wait 5s after scan before checking for face departure, + // so the greeting has time to play. + grace_period_sec: 5.0, + }, + }, + ], + lifecycle_hooks: [ + { + // Tell approaching guests how to check in as soon as this mode is entered. + hook_type: "on_entry", + handler_type: "function", + handler_config: { + module_name: "greeting_hook", + function: "greeting_start_hook", + tts_provider: "elevenlabs", + custom_message: "Hi there! Welcome to our event. Are you registered on Luma?", + voice_id: "${VOICE_ID:-PoHUWWWMHFrA8z7Q88pu}", + }, + }, + ], + }, + }, + + transition_rules: [ + { + // Switch to check-in once a face is close enough (published by FaceSizeWatch). + from_mode: "face_watch", + to_mode: "checkin", + transition_type: "context_aware", + context_conditions: { face_close_enough: true }, + priority: 0, + cooldown_seconds: 3.0, }, { - name: "emotion", - llm_label: "emotion", - connector: "zenoh", + // Return to watching after a successful check-in (published by the speak action + // or a dedupe-window expiry — reuse the greeting_conversation_finished signal + // or add a dedicated one as needed). + from_mode: "checkin", + to_mode: "face_watch", + transition_type: "context_aware", + context_conditions: { checkin_complete: true }, + priority: 0, + cooldown_seconds: 5.0, }, ], } diff --git a/plugins/backgrounds/checkin_complete.go b/plugins/backgrounds/checkin_complete.go new file mode 100644 index 0000000000..2ac21e4a44 --- /dev/null +++ b/plugins/backgrounds/checkin_complete.go @@ -0,0 +1,139 @@ +package backgrounds + +import ( + "context" + "encoding/json" + "strings" + "time" + + "go.uber.org/zap" + + bg "github.com/openmind/om1/internal/backgrounds" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers" + "github.com/openmind/om1/internal/util" +) + +func init() { + bg.Register("CheckinComplete", NewCheckinComplete) +} + +type checkinCompleteConfig struct { + FaceBaseURL string `json:"face_http_base_url"` + FaceRecentSec float64 `json:"face_recent_sec"` + FaceMinArea float64 `json:"min_face_area"` + PollSec float64 `json:"poll_interval_sec"` + ScanIOKey string `json:"scan_io_key"` + GracePeriodSec float64 `json:"grace_period_sec"` +} + +// CheckinComplete publishes checkin_complete:true to ModeContext once a +// successful QR scan has been recorded AND the guest's face is no longer +// visible (or too small). +type CheckinComplete struct { + log *zap.Logger + face *providers.FacePresenceProvider + period time.Duration + scanIOKey string + gracePeriod time.Duration + scanSeen bool + scanTime time.Time +} + +func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { + var cfg checkinCompleteConfig + if b, err := json.Marshal(configMap); err == nil { + _ = json.Unmarshal(b, &cfg) + } + if cfg.FaceBaseURL == "" { + cfg.FaceBaseURL = "http://127.0.0.1:6793" + } + if cfg.FaceRecentSec <= 0 { + cfg.FaceRecentSec = 1.0 + } + if cfg.FaceMinArea <= 0 { + cfg.FaceMinArea = 15000 + } + if cfg.PollSec <= 0 { + cfg.PollSec = 1.0 + } + if cfg.ScanIOKey == "" { + cfg.ScanIOKey = "QRScannerRTSP" + } + if cfg.GracePeriodSec <= 0 { + cfg.GracePeriodSec = 5.0 + } + + log := logger.Get().Named("CheckinComplete") + + face := providers.NewFacePresenceProvider(providers.FacePresenceConfig{ + BaseURL: cfg.FaceBaseURL, + RecentSec: cfg.FaceRecentSec, + Timeout: 2 * time.Second, + MinFaceArea: cfg.FaceMinArea, + }) + + log.Info("initialized", + zap.String("scan_io_key", cfg.ScanIOKey), + zap.Float64("grace_period_sec", cfg.GracePeriodSec), + ) + + return &CheckinComplete{ + log: log, + face: face, + period: time.Duration(cfg.PollSec * float64(time.Second)), + scanIOKey: cfg.ScanIOKey, + gracePeriod: time.Duration(cfg.GracePeriodSec * float64(time.Second)), + }, nil +} + +func (c *CheckinComplete) Run(ctx context.Context) { + // Step 1: check if a successful scan has been recorded. + if !c.scanSeen { + if in := providers.IO().GetInput(c.scanIOKey); in != nil { + if strings.HasPrefix(in.Input, "qr_scan: name=") { + c.scanSeen = true + c.scanTime = in.Timestamp + c.log.Info("successful scan detected", zap.String("value", in.Input)) + } + } + } + + if !c.scanSeen { + util.Sleep(ctx, c.period) + return + } + + // Step 2: wait a grace period after the scan before checking for face + // departure, so the greeting has time to play. + if time.Since(c.scanTime) < c.gracePeriod { + util.Sleep(ctx, c.period) + return + } + + // Step 3: check if the face is gone. + snap, err := c.face.FetchSnapshot(ctx) + if err != nil { + if ctx.Err() == nil { + c.log.Warn("failed to fetch face snapshot", zap.Error(err)) + } + util.Sleep(ctx, c.period) + return + } + + totalFaces := len(snap.Names) + snap.UnknownFaces + if totalFaces > 0 { + util.Sleep(ctx, c.period) + return + } + + c.log.Info("guest departed after successful scan, triggering transition") + providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) + c.scanSeen = false + + util.Sleep(ctx, c.period) +} + +func (c *CheckinComplete) Stop() { + c.log.Info("stopping") +} diff --git a/plugins/backgrounds/face_size_watch.go b/plugins/backgrounds/face_size_watch.go new file mode 100644 index 0000000000..4fc147d731 --- /dev/null +++ b/plugins/backgrounds/face_size_watch.go @@ -0,0 +1,99 @@ +package backgrounds + +import ( + "context" + "encoding/json" + "time" + + "go.uber.org/zap" + + bg "github.com/openmind/om1/internal/backgrounds" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers" + "github.com/openmind/om1/internal/util" +) + +func init() { + bg.Register("FaceSizeWatch", NewFaceSizeWatch) +} + +type faceSizeWatchConfig struct { + BaseURL string `json:"face_http_base_url"` + RecentSec float64 `json:"face_recent_sec"` + PollSec float64 `json:"face_poll_interval_sec"` + MinFaceArea float64 `json:"min_face_area"` +} + +type FaceSizeWatch struct { + log *zap.Logger + provider *providers.FacePresenceProvider + period time.Duration +} + +func NewFaceSizeWatch(configMap map[string]any) (bg.Background, error) { + var cfg faceSizeWatchConfig + if b, err := json.Marshal(configMap); err == nil { + _ = json.Unmarshal(b, &cfg) + } + if cfg.BaseURL == "" { + cfg.BaseURL = "http://127.0.0.1:6793" + } + if cfg.RecentSec <= 0 { + cfg.RecentSec = 1.0 + } + if cfg.PollSec <= 0 { + cfg.PollSec = 0.5 + } + if cfg.MinFaceArea <= 0 { + cfg.MinFaceArea = 15000 + } + + log := logger.Get().Named("FaceSizeWatch") + + provider := providers.NewFacePresenceProvider(providers.FacePresenceConfig{ + BaseURL: cfg.BaseURL, + RecentSec: cfg.RecentSec, + Timeout: 2 * time.Second, + MinFaceArea: cfg.MinFaceArea, + }) + + log.Info("initialized", + zap.String("base_url", cfg.BaseURL), + zap.Float64("min_face_area", cfg.MinFaceArea), + zap.Float64("poll_sec", cfg.PollSec), + ) + + return &FaceSizeWatch{ + log: log, + provider: provider, + period: time.Duration(cfg.PollSec * float64(time.Second)), + }, nil +} + +func (f *FaceSizeWatch) Run(ctx context.Context) { + snap, err := f.provider.FetchSnapshot(ctx) + if err != nil { + if ctx.Err() == nil { + f.log.Warn("failed to fetch snapshot", zap.Error(err)) + } + util.Sleep(ctx, f.period) + return + } + + // FetchSnapshot already filters by MinFaceArea, so any face in the + // snapshot is large enough. Count known + unknown faces. + totalFaces := len(snap.Names) + snap.UnknownFaces + if totalFaces > 0 { + f.log.Info("face close enough, triggering transition", + zap.Int("faces", totalFaces), + zap.String("closest", snap.ClosestName), + ) + providers.ModeContext().Publish(map[string]any{"face_close_enough": true}) + } + + util.Sleep(ctx, f.period) +} + +func (f *FaceSizeWatch) Stop() { + f.log.Info("stopping") +} From 2dbe86df7e76a8d000d43b4933ef53314eb185cc Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 14:19:01 -0700 Subject: [PATCH 20/40] remove config --- config/luma_checkin.json5 | 188 -------------------------------- config/luma_checkin_local.json5 | 47 -------- 2 files changed, 235 deletions(-) delete mode 100644 config/luma_checkin.json5 delete mode 100644 config/luma_checkin_local.json5 diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 deleted file mode 100644 index 2e9bb9c96b..0000000000 --- a/config/luma_checkin.json5 +++ /dev/null @@ -1,188 +0,0 @@ -{ - version: "v1.0.5", - name: "luma_checkin_go2", - default_mode: "face_watch", - allow_manual_switching: true, - - api_key: "${OM_API_KEY:-openmind_free}", - system_governance: "", - - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - - modes: { - // Mode 1: Passive face-size watch. Waits until a face bounding box area - // exceeds min_face_area pixels, then transitions to checkin mode. - face_watch: { - display_name: "Face Watch Mode", - description: "Monitors camera for a face close enough to check in. Transitions to checkin once a large-enough face is detected.", - system_prompt_base: "", - hertz: 0.5, - agent_inputs: [], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [], - // FaceSizeWatch polls the face HTTP service and publishes - // face_close_enough: true to ModeContext when a bounding box - // area exceeds min_face_area pixels. - backgrounds: [ - { - type: "FaceSizeWatch", - config: { - face_http_base_url: "http://127.0.0.1:6793", - face_recent_sec: 1.0, - face_poll_interval_sec: 0.5, - min_face_area: 15000, - }, - }, - ], - lifecycle_hooks: [], - }, - - // Mode 2: Active check-in. Announces instructions on entry, runs the QR - // scanner in the background, and lets the LLM speak the personalized - // greeting when a scan arrives. The QR scanner triggers an immediate tick - // (bypassing the hertz timer) on every fresh scan so the LLM responds - // without waiting for the next scheduled cycle. - checkin: { - display_name: "Check-in Mode", - description: "Greets guests by QR code. Announces instructions on entry and speaks the greeting returned by the Luma lookup.", - system_prompt_base: "You are a friendly event check-in robot.\n\ -\n\ -Check-in flow:\n\ -1. First ask the guest: 'Hi there! Are you registered on Luma for this event?'\n\ -2. If they say YES (or similar): say 'Great! Please hold up your QR code to the camera and I will check you in.'\n\ - Then wait for the QR Scanner to pick it up.\n\ -3. If they say NO (or not registered, or similar): say 'No problem! Could you tell me your name so I can check you in?'\n\ - When they give their name, welcome them warmly by name.\n\ -\n\ -QR Scanner handling:\n\ -The QR Scanner input emits one of these messages when it sees a check-in QR:\n\ - - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n\ - - 'qr_scan_failed: pk= reason=' on a failure.\n\ -\n\ -On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text — do not paraphrase or invent names.\n\ -On 'qr_scan_failed:': apologize and ask them to try again or tell you their name instead.\n\ -If there is no qr_scan line in the current observations, continue the conversation naturally.\n\ -Do not invent scan results. Do not narrate.", - hertz: 0.5, - agent_inputs: [ - { - type: "${ASR_INPUT_PLUGIN:-GoogleASRInput}", - config: { - api_version: "v2", - enable_tts_interrupt: false, - rate: 16000, - chunk: 1600, - }, - }, - { - type: "ConversationHistoryInput", - config: { - max_rounds: 5, - }, - }, - { - type: "QRScannerRTSP", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 5, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, - }, - { - name: "emotion", - llm_label: "emotion", - connector: "zenoh", - }, - ], - backgrounds: [ - { - type: "CheckinComplete", - config: { - face_http_base_url: "http://127.0.0.1:6793", - min_face_area: 15000, - poll_interval_sec: 1.0, - scan_io_key: "QRScannerRTSP", - // Wait 5s after scan before checking for face departure, - // so the greeting has time to play. - grace_period_sec: 5.0, - }, - }, - ], - lifecycle_hooks: [ - { - // Tell approaching guests how to check in as soon as this mode is entered. - hook_type: "on_entry", - handler_type: "function", - handler_config: { - module_name: "greeting_hook", - function: "greeting_start_hook", - tts_provider: "elevenlabs", - custom_message: "Hi there! Welcome to our event. Are you registered on Luma?", - voice_id: "${VOICE_ID:-PoHUWWWMHFrA8z7Q88pu}", - }, - }, - ], - }, - }, - - transition_rules: [ - { - // Switch to check-in once a face is close enough (published by FaceSizeWatch). - from_mode: "face_watch", - to_mode: "checkin", - transition_type: "context_aware", - context_conditions: { face_close_enough: true }, - priority: 0, - cooldown_seconds: 3.0, - }, - { - // Return to watching after a successful check-in (published by the speak action - // or a dedupe-window expiry — reuse the greeting_conversation_finished signal - // or add a dedicated one as needed). - from_mode: "checkin", - to_mode: "face_watch", - transition_type: "context_aware", - context_conditions: { checkin_complete: true }, - priority: 0, - cooldown_seconds: 5.0, - }, - ], -} diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 deleted file mode 100644 index b56a011735..0000000000 --- a/config/luma_checkin_local.json5 +++ /dev/null @@ -1,47 +0,0 @@ -{ - version: "v1.0.5", - hertz: 0.2, - name: "luma_checkin_local", - api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", - system_governance: "", - agent_inputs: [ - { - type: "QRScanner", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - resolution_width: 1280, - resolution_height: 720, - jpeg_quality: 60, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - implementation: "passthrough", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, - }, - ], -} From 1e06f7aa7136bcc7d305afd5f0361148affb4935 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 14:33:50 -0700 Subject: [PATCH 21/40] update face check area --- config/luma_checkin.json5 | 188 ++++++++++++++++++++++++ config/luma_checkin_local.json5 | 47 ++++++ plugins/backgrounds/checkin_complete.go | 2 +- plugins/backgrounds/face_size_watch.go | 2 +- 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 config/luma_checkin.json5 create mode 100644 config/luma_checkin_local.json5 diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 new file mode 100644 index 0000000000..c5cb5a7520 --- /dev/null +++ b/config/luma_checkin.json5 @@ -0,0 +1,188 @@ +{ + version: "v1.0.5", + name: "luma_checkin_go2", + default_mode: "face_watch", + allow_manual_switching: true, + + api_key: "${OM_API_KEY:-openmind_free}", + system_governance: "", + + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 0, + }, + }, + + modes: { + // Mode 1: Passive face-size watch. Waits until a face bounding box area + // exceeds min_face_area pixels, then transitions to checkin mode. + face_watch: { + display_name: "Face Watch Mode", + description: "Monitors camera for a face close enough to check in. Transitions to checkin once a large-enough face is detected.", + system_prompt_base: "", + hertz: 0.5, + agent_inputs: [], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 0, + }, + }, + action_execution_mode: "concurrent", + agent_actions: [], + // FaceSizeWatch polls the face HTTP service and publishes + // face_close_enough: true to ModeContext when a bounding box + // area exceeds min_face_area pixels. + backgrounds: [ + { + type: "FaceSizeWatch", + config: { + face_http_base_url: "http://127.0.0.1:6793", + face_recent_sec: 1.0, + face_poll_interval_sec: 0.5, + min_face_area: 3000, + }, + }, + ], + lifecycle_hooks: [], + }, + + // Mode 2: Active check-in. Announces instructions on entry, runs the QR + // scanner in the background, and lets the LLM speak the personalized + // greeting when a scan arrives. The QR scanner triggers an immediate tick + // (bypassing the hertz timer) on every fresh scan so the LLM responds + // without waiting for the next scheduled cycle. + checkin: { + display_name: "Check-in Mode", + description: "Greets guests by QR code. Announces instructions on entry and speaks the greeting returned by the Luma lookup.", + system_prompt_base: "You are a friendly event check-in robot.\n\ +\n\ +Check-in flow:\n\ +1. First ask the guest: 'Hi there! Are you registered on Luma for this event?'\n\ +2. If they say YES (or similar): say 'Great! Please hold up your QR code to the camera and I will check you in.'\n\ + Then wait for the QR Scanner to pick it up.\n\ +3. If they say NO (or not registered, or similar): say 'No problem! Could you tell me your name so I can check you in?'\n\ + When they give their name, welcome them warmly by name.\n\ +\n\ +QR Scanner handling:\n\ +The QR Scanner input emits one of these messages when it sees a check-in QR:\n\ + - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n\ + - 'qr_scan_failed: pk= reason=' on a failure.\n\ +\n\ +On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text — do not paraphrase or invent names.\n\ +On 'qr_scan_failed:': apologize and ask them to try again or tell you their name instead.\n\ +If there is no qr_scan line in the current observations, continue the conversation naturally.\n\ +Do not invent scan results. Do not narrate.", + hertz: 0.5, + agent_inputs: [ + { + type: "${ASR_INPUT_PLUGIN:-GoogleASRInput}", + config: { + api_version: "v2", + enable_tts_interrupt: false, + rate: 16000, + chunk: 1600, + }, + }, + { + type: "ConversationHistoryInput", + config: { + max_rounds: 5, + }, + }, + { + type: "QRScannerRTSP", + config: { + camera_index: 0, + capture_fps: 30, + decode_fps: 5, + dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", + request_timeout_seconds: 4, + }, + }, + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 5, + }, + }, + action_execution_mode: "concurrent", + agent_actions: [ + { + name: "speak", + llm_label: "speak", + connector: "elevenlabs_tts", + config: { + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + { + name: "emotion", + llm_label: "emotion", + connector: "zenoh", + }, + ], + backgrounds: [ + { + type: "CheckinComplete", + config: { + face_http_base_url: "http://127.0.0.1:6793", + min_face_area: 3000, + poll_interval_sec: 1.0, + scan_io_key: "QRScannerRTSP", + // Wait 5s after scan before checking for face departure, + // so the greeting has time to play. + grace_period_sec: 5.0, + }, + }, + ], + lifecycle_hooks: [ + { + // Tell approaching guests how to check in as soon as this mode is entered. + hook_type: "on_entry", + handler_type: "function", + handler_config: { + module_name: "greeting_hook", + function: "greeting_start_hook", + tts_provider: "elevenlabs", + custom_message: "Hi there! Welcome to our event. Are you registered on Luma?", + voice_id: "${VOICE_ID:-PoHUWWWMHFrA8z7Q88pu}", + }, + }, + ], + }, + }, + + transition_rules: [ + { + // Switch to check-in once a face is close enough (published by FaceSizeWatch). + from_mode: "face_watch", + to_mode: "checkin", + transition_type: "context_aware", + context_conditions: { face_close_enough: true }, + priority: 0, + cooldown_seconds: 3.0, + }, + { + // Return to watching after a successful check-in (published by the speak action + // or a dedupe-window expiry — reuse the greeting_conversation_finished signal + // or add a dedicated one as needed). + from_mode: "checkin", + to_mode: "face_watch", + transition_type: "context_aware", + context_conditions: { checkin_complete: true }, + priority: 0, + cooldown_seconds: 5.0, + }, + ], +} diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 new file mode 100644 index 0000000000..b56a011735 --- /dev/null +++ b/config/luma_checkin_local.json5 @@ -0,0 +1,47 @@ +{ + version: "v1.0.5", + hertz: 0.2, + name: "luma_checkin_local", + api_key: "${OM_API_KEY:-openmind_free}", + system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", + system_governance: "", + agent_inputs: [ + { + type: "QRScanner", + config: { + camera_index: 0, + capture_fps: 30, + decode_fps: 5, + resolution_width: 1280, + resolution_height: 720, + jpeg_quality: 60, + dedupe_window_seconds: 30, + luma: { + api_key: "${LUMA_API_KEY}", + event_api_id: "${LUMA_EVENT_ID}", + greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", + request_timeout_seconds: 4, + }, + }, + }, + ], + cortex_llm: { + type: "GeminiLLM", + config: { + agent_name: "Greeter", + history_length: 0, + }, + }, + action_execution_mode: "concurrent", + agent_actions: [ + { + name: "speak", + llm_label: "speak", + implementation: "passthrough", + connector: "elevenlabs_tts", + config: { + voice_id: "PoHUWWWMHFrA8z7Q88pu", + }, + }, + ], +} diff --git a/plugins/backgrounds/checkin_complete.go b/plugins/backgrounds/checkin_complete.go index 2ac21e4a44..a381732c07 100644 --- a/plugins/backgrounds/checkin_complete.go +++ b/plugins/backgrounds/checkin_complete.go @@ -52,7 +52,7 @@ func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { cfg.FaceRecentSec = 1.0 } if cfg.FaceMinArea <= 0 { - cfg.FaceMinArea = 15000 + cfg.FaceMinArea = 3000 } if cfg.PollSec <= 0 { cfg.PollSec = 1.0 diff --git a/plugins/backgrounds/face_size_watch.go b/plugins/backgrounds/face_size_watch.go index 4fc147d731..2dbc226635 100644 --- a/plugins/backgrounds/face_size_watch.go +++ b/plugins/backgrounds/face_size_watch.go @@ -45,7 +45,7 @@ func NewFaceSizeWatch(configMap map[string]any) (bg.Background, error) { cfg.PollSec = 0.5 } if cfg.MinFaceArea <= 0 { - cfg.MinFaceArea = 15000 + cfg.MinFaceArea = 3000 } log := logger.Get().Named("FaceSizeWatch") From 8b009d8f8a972e362b08b19b16c44b5178fe9db6 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 14:34:47 -0700 Subject: [PATCH 22/40] remove config --- config/luma_checkin.json5 | 188 -------------------------------- config/luma_checkin_local.json5 | 47 -------- 2 files changed, 235 deletions(-) delete mode 100644 config/luma_checkin.json5 delete mode 100644 config/luma_checkin_local.json5 diff --git a/config/luma_checkin.json5 b/config/luma_checkin.json5 deleted file mode 100644 index c5cb5a7520..0000000000 --- a/config/luma_checkin.json5 +++ /dev/null @@ -1,188 +0,0 @@ -{ - version: "v1.0.5", - name: "luma_checkin_go2", - default_mode: "face_watch", - allow_manual_switching: true, - - api_key: "${OM_API_KEY:-openmind_free}", - system_governance: "", - - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - - modes: { - // Mode 1: Passive face-size watch. Waits until a face bounding box area - // exceeds min_face_area pixels, then transitions to checkin mode. - face_watch: { - display_name: "Face Watch Mode", - description: "Monitors camera for a face close enough to check in. Transitions to checkin once a large-enough face is detected.", - system_prompt_base: "", - hertz: 0.5, - agent_inputs: [], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [], - // FaceSizeWatch polls the face HTTP service and publishes - // face_close_enough: true to ModeContext when a bounding box - // area exceeds min_face_area pixels. - backgrounds: [ - { - type: "FaceSizeWatch", - config: { - face_http_base_url: "http://127.0.0.1:6793", - face_recent_sec: 1.0, - face_poll_interval_sec: 0.5, - min_face_area: 3000, - }, - }, - ], - lifecycle_hooks: [], - }, - - // Mode 2: Active check-in. Announces instructions on entry, runs the QR - // scanner in the background, and lets the LLM speak the personalized - // greeting when a scan arrives. The QR scanner triggers an immediate tick - // (bypassing the hertz timer) on every fresh scan so the LLM responds - // without waiting for the next scheduled cycle. - checkin: { - display_name: "Check-in Mode", - description: "Greets guests by QR code. Announces instructions on entry and speaks the greeting returned by the Luma lookup.", - system_prompt_base: "You are a friendly event check-in robot.\n\ -\n\ -Check-in flow:\n\ -1. First ask the guest: 'Hi there! Are you registered on Luma for this event?'\n\ -2. If they say YES (or similar): say 'Great! Please hold up your QR code to the camera and I will check you in.'\n\ - Then wait for the QR Scanner to pick it up.\n\ -3. If they say NO (or not registered, or similar): say 'No problem! Could you tell me your name so I can check you in?'\n\ - When they give their name, welcome them warmly by name.\n\ -\n\ -QR Scanner handling:\n\ -The QR Scanner input emits one of these messages when it sees a check-in QR:\n\ - - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n\ - - 'qr_scan_failed: pk= reason=' on a failure.\n\ -\n\ -On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text — do not paraphrase or invent names.\n\ -On 'qr_scan_failed:': apologize and ask them to try again or tell you their name instead.\n\ -If there is no qr_scan line in the current observations, continue the conversation naturally.\n\ -Do not invent scan results. Do not narrate.", - hertz: 0.5, - agent_inputs: [ - { - type: "${ASR_INPUT_PLUGIN:-GoogleASRInput}", - config: { - api_version: "v2", - enable_tts_interrupt: false, - rate: 16000, - chunk: 1600, - }, - }, - { - type: "ConversationHistoryInput", - config: { - max_rounds: 5, - }, - }, - { - type: "QRScannerRTSP", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 5, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, - }, - { - name: "emotion", - llm_label: "emotion", - connector: "zenoh", - }, - ], - backgrounds: [ - { - type: "CheckinComplete", - config: { - face_http_base_url: "http://127.0.0.1:6793", - min_face_area: 3000, - poll_interval_sec: 1.0, - scan_io_key: "QRScannerRTSP", - // Wait 5s after scan before checking for face departure, - // so the greeting has time to play. - grace_period_sec: 5.0, - }, - }, - ], - lifecycle_hooks: [ - { - // Tell approaching guests how to check in as soon as this mode is entered. - hook_type: "on_entry", - handler_type: "function", - handler_config: { - module_name: "greeting_hook", - function: "greeting_start_hook", - tts_provider: "elevenlabs", - custom_message: "Hi there! Welcome to our event. Are you registered on Luma?", - voice_id: "${VOICE_ID:-PoHUWWWMHFrA8z7Q88pu}", - }, - }, - ], - }, - }, - - transition_rules: [ - { - // Switch to check-in once a face is close enough (published by FaceSizeWatch). - from_mode: "face_watch", - to_mode: "checkin", - transition_type: "context_aware", - context_conditions: { face_close_enough: true }, - priority: 0, - cooldown_seconds: 3.0, - }, - { - // Return to watching after a successful check-in (published by the speak action - // or a dedupe-window expiry — reuse the greeting_conversation_finished signal - // or add a dedicated one as needed). - from_mode: "checkin", - to_mode: "face_watch", - transition_type: "context_aware", - context_conditions: { checkin_complete: true }, - priority: 0, - cooldown_seconds: 5.0, - }, - ], -} diff --git a/config/luma_checkin_local.json5 b/config/luma_checkin_local.json5 deleted file mode 100644 index b56a011735..0000000000 --- a/config/luma_checkin_local.json5 +++ /dev/null @@ -1,47 +0,0 @@ -{ - version: "v1.0.5", - hertz: 0.2, - name: "luma_checkin_local", - api_key: "${OM_API_KEY:-openmind_free}", - system_prompt_base: "You are a friendly event greeter robot. The QR Scanner input emits one of these messages when it sees a check-in QR:\n - 'qr_scan: name= greeting=\"\"' on a successful guest lookup.\n - 'qr_scan_failed: pk= reason=' on a failure.\n - 'qr_scan: pk= event=' if Luma lookup is disabled (no enrichment).\n\nOnly act when the CURRENT tick has one of these lines under 'Current observations'. On 'qr_scan: name=...': call speak with EXACTLY the quoted greeting text from the observation as the action argument — do not paraphrase, do not invent names. On 'qr_scan_failed:' or a bare 'qr_scan: pk=...' (no name): do nothing. If 'Current observations' is empty or has no qr_scan line, do nothing — do not call any action. Do not narrate.", - system_governance: "", - agent_inputs: [ - { - type: "QRScanner", - config: { - camera_index: 0, - capture_fps: 30, - decode_fps: 5, - resolution_width: 1280, - resolution_height: 720, - jpeg_quality: 60, - dedupe_window_seconds: 30, - luma: { - api_key: "${LUMA_API_KEY}", - event_api_id: "${LUMA_EVENT_ID}", - greeting_template: "Welcome, {first_name}! I'm so glad to see you here today at our PopUp Store. I hope you have a great day.", - request_timeout_seconds: 4, - }, - }, - }, - ], - cortex_llm: { - type: "GeminiLLM", - config: { - agent_name: "Greeter", - history_length: 0, - }, - }, - action_execution_mode: "concurrent", - agent_actions: [ - { - name: "speak", - llm_label: "speak", - implementation: "passthrough", - connector: "elevenlabs_tts", - config: { - voice_id: "PoHUWWWMHFrA8z7Q88pu", - }, - }, - ], -} From 2712e7c21425e9a235d007fcb2e8385649c3056d Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Fri, 12 Jun 2026 18:41:03 -0700 Subject: [PATCH 23/40] revert vlm video dtream changes --- config/conversation.json5 | 5 -- internal/providers/vlm/video_stream.go | 63 ++++---------------------- internal/providers/vlm/vlm_test.go | 6 +-- 3 files changed, 11 insertions(+), 63 deletions(-) diff --git a/config/conversation.json5 b/config/conversation.json5 index ddbafa548b..aaa14a50c1 100644 --- a/config/conversation.json5 +++ b/config/conversation.json5 @@ -22,11 +22,6 @@ }, { type: "VLMGemini", - config: { - resolution_width: 1280, - resolution_height: 720, - fps: 30, - }, } ], cortex_llm: { diff --git a/internal/providers/vlm/video_stream.go b/internal/providers/vlm/video_stream.go index 3ce9b31990..ee83eff3c1 100644 --- a/internal/providers/vlm/video_stream.go +++ b/internal/providers/vlm/video_stream.go @@ -1,10 +1,8 @@ package vlm import ( - "bufio" "context" "fmt" - "io" "os/exec" "runtime" "strconv" @@ -17,9 +15,9 @@ import ( ) const ( + defaultFPS = 30 defaultWidth = 640 defaultHeight = 480 - defaultFPS = 30 defaultJPEGQuality = 30 cameraRetryDelay = 2 * time.Second ) @@ -105,17 +103,10 @@ func (v *VideoStream) stream(ctx context.Context, cam string) error { return fmt.Errorf("stdout pipe: %w", err) } - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("stderr pipe: %w", err) - } - if err := cmd.Start(); err != nil { return fmt.Errorf("start ffmpeg: %w", err) } - go v.logFFmpegStderr(stderr) - defer func() { if cmd.Process != nil { _ = cmd.Process.Kill() @@ -135,17 +126,6 @@ func (v *VideoStream) stream(ctx context.Context, cam string) error { }) } -func (v *VideoStream) logFFmpegStderr(r io.ReadCloser) { - defer r.Close() - scanner := bufio.NewScanner(r) - for scanner.Scan() { - v.log.Warn("VideoStream: ffmpeg", zap.String("stderr", scanner.Text())) - } - if err := scanner.Err(); err != nil { - v.log.Warn("VideoStream: ffmpeg stderr read error", zap.Error(err)) - } -} - // cameraInput returns the ffmpeg input specifier for the configured camera device. func (v *VideoStream) cameraInput() string { if runtime.GOOS == "darwin" { @@ -157,48 +137,21 @@ func (v *VideoStream) cameraInput() string { // ffmpegArgs constructs the ffmpeg command-line arguments for the configured stream. func (v *VideoStream) ffmpegArgs(cam string) []string { + inputFormat := "v4l2" if runtime.GOOS == "darwin" { - // avfoundation pairs framerate to the chosen video size, so only emit - // -framerate when -video_size is also set; otherwise ffmpeg picks a - // default size whose mode list may not include the requested rate. - // Pixel format is left unset so avfoundation picks the camera's native - // format (e.g. nv12 on Apple Silicon / Continuity cameras). - args := []string{ - "-loglevel", "error", - "-f", "avfoundation", - } - if v.cfg.Width > 0 && v.cfg.Height > 0 { - args = append(args, "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height)) - if v.cfg.FPS > 0 { - args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) - } - } - return append(args, - "-i", cam, - "-an", - "-c:v", "mjpeg", - "-qscale:v", strconv.Itoa(jpegQScale(v.cfg.JPEGQuality)), - "-f", "image2pipe", - "pipe:1", - ) + inputFormat = "avfoundation" } - args := []string{ + return []string{ "-loglevel", "error", - "-f", "v4l2", - } - if v.cfg.FPS > 0 { - args = append(args, "-framerate", strconv.Itoa(v.cfg.FPS)) - } - if v.cfg.Width > 0 && v.cfg.Height > 0 { - args = append(args, "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height)) - } - return append(args, + "-f", inputFormat, + "-framerate", strconv.Itoa(v.cfg.FPS), + "-video_size", fmt.Sprintf("%dx%d", v.cfg.Width, v.cfg.Height), "-i", cam, "-an", "-c:v", "mjpeg", "-qscale:v", strconv.Itoa(jpegQScale(v.cfg.JPEGQuality)), "-f", "image2pipe", "pipe:1", - ) + } } diff --git a/internal/providers/vlm/vlm_test.go b/internal/providers/vlm/vlm_test.go index 261147564e..da72652621 100644 --- a/internal/providers/vlm/vlm_test.go +++ b/internal/providers/vlm/vlm_test.go @@ -141,9 +141,9 @@ func TestParseAVFoundationDevices(t *testing.T) { func TestNewVideoStreamDefaults(t *testing.T) { v := NewVideoStream(VideoStreamConfig{}) - assert.Equal(t, 0, v.cfg.FPS) - assert.Equal(t, 0, v.cfg.Width) - assert.Equal(t, 0, v.cfg.Height) + assert.Equal(t, defaultFPS, v.cfg.FPS) + assert.Equal(t, defaultWidth, v.cfg.Width) + assert.Equal(t, defaultHeight, v.cfg.Height) assert.Equal(t, defaultJPEGQuality, v.cfg.JPEGQuality) } From 1eb0478ba1592bd99e4e3355b4ebf927bd60c524 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Sat, 13 Jun 2026 20:32:14 -0700 Subject: [PATCH 24/40] added luma checkin flow --- internal/providers/luma/client.go | 64 ++++++++++++++++++++++-- plugins/inputs/qr_scanner/export_test.go | 4 ++ plugins/inputs/qr_scanner/qr_scanner.go | 17 +++++-- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go index 05b51a07e8..3a0ef56d9b 100644 --- a/internal/providers/luma/client.go +++ b/internal/providers/luma/client.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "bytes" "io" "net/http" "net/url" @@ -18,8 +19,9 @@ import ( ) const ( - DefaultBaseURL = "https://public-api.luma.com" - GetGuestPath = "/v1/event/get-guest" + DefaultBaseURL = "https://public-api.luma.com" + GetGuestPath = "/v1/event/get-guest" + CheckInURL = "https://api.luma.com/event/admin/update-check-in" ) var ( @@ -60,24 +62,33 @@ type Client struct { baseURL string apiKey string eventAPIID string + sessionKey string http HTTPDoer timeout time.Duration } -func NewClient(baseURL, apiKey, eventAPIID string, timeout time.Duration) *Client { +func NewClient(baseURL, apiKey, eventAPIID string, timeout time.Duration, opts ...func(*Client)) *Client { if baseURL == "" { baseURL = DefaultBaseURL } if timeout <= 0 { timeout = 5 * time.Second } - return &Client{ + c := &Client{ baseURL: strings.TrimRight(baseURL, "/"), apiKey: apiKey, eventAPIID: eventAPIID, http: httpclient.Default(), timeout: timeout, } + for _, o := range opts { + o(c) + } + return c +} + +func WithSessionKey(key string) func(*Client) { + return func(c *Client) { c.sessionKey = key } } // SetHTTPDoer overrides the underlying http client. Intended for tests. @@ -175,6 +186,51 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { return c.http.Do(req) } +// CheckIn marks a guest as checked-in via Luma's admin endpoint. +// Requires a session key (from a logged-in browser session). +func (c *Client) CheckIn(ctx context.Context, guest *Guest) error { + if c.sessionKey == "" { + return fmt.Errorf("luma check-in: no session key configured") + } + if guest == nil || guest.APIID == "" { + return fmt.Errorf("luma check-in: missing guest api_id") + } + + payload := map[string]string{ + "event_api_id": c.eventAPIID, + "rsvp_api_id": guest.APIID, + "check_in_method": "guest-list", + "check_in_status": "checked-in", + "type": "guest", + } + body, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("luma check-in marshal: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, CheckInURL, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("luma check-in request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Cookie", "luma.auth-session-key="+c.sessionKey) + + resp, err := c.do(req) + if err != nil { + return fmt.Errorf("luma check-in: %w", err) + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("luma check-in %d: %s", resp.StatusCode, string(respBody)) + } + return nil +} + // FirstName picks the best available first-name field from a Guest record. // Falls back to "friend" when nothing usable is set. func FirstName(g *Guest) string { diff --git a/plugins/inputs/qr_scanner/export_test.go b/plugins/inputs/qr_scanner/export_test.go index 827ef8aa93..5c53e34fc0 100644 --- a/plugins/inputs/qr_scanner/export_test.go +++ b/plugins/inputs/qr_scanner/export_test.go @@ -61,3 +61,7 @@ type FakeLookup struct { func (f *FakeLookup) GetGuest(_ context.Context, _ string) (*luma.Guest, error) { return f.Guest, f.Err } + +func (f *FakeLookup) CheckIn(_ context.Context, _ *luma.Guest) error { + return nil +} diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/qr_scanner/qr_scanner.go index b0de78fc53..1be204c0b7 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/qr_scanner/qr_scanner.go @@ -45,6 +45,7 @@ type LumaConfig struct { EventAPIID string `json:"event_api_id"` GreetingTemplate string `json:"greeting_template"` RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` + SessionKey string `json:"session_key"` } // SpeakConfig, when set, makes the scanner push the resolved greeting straight @@ -79,10 +80,9 @@ type frameSource interface { Stop() } -// guestLookup is the slice of luma.Client behavior the scanner uses; -// abstracted for tests. type guestLookup interface { GetGuest(ctx context.Context, pk string) (*luma.Guest, error) + CheckIn(ctx context.Context, guest *luma.Guest) error } // ttsSpeaker is the slice of *tts.ElevenLabsProvider the scanner uses to @@ -172,7 +172,11 @@ func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { if timeout <= 0 { timeout = defaultLumaTimeout } - s.luma = luma.NewClient(cfg.Luma.BaseURL, cfg.Luma.APIKey, cfg.Luma.EventAPIID, timeout) + var opts []func(*luma.Client) + if cfg.Luma.SessionKey != "" { + opts = append(opts, luma.WithSessionKey(cfg.Luma.SessionKey)) + } + s.luma = luma.NewClient(cfg.Luma.BaseURL, cfg.Luma.APIKey, cfg.Luma.EventAPIID, timeout, opts...) s.lumaTimeout = timeout s.expectedEventID = cfg.Luma.EventAPIID s.greetingTmpl = cfg.Luma.GreetingTemplate @@ -421,6 +425,13 @@ func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) stri name := luma.FirstName(guest) greeting := luma.FormatGreeting(s.greetingTmpl, guest) + + if err := s.luma.CheckIn(ctx, guest); err != nil { + s.log.Warn("luma check-in failed", zap.String("pk", pk), zap.Error(err)) + } else { + s.log.Info("luma check-in ok", zap.String("pk", pk), zap.String("name", name)) + } + if s.speak != nil { s.speak.AddText(greeting) s.log.Info("greeting pushed to tts", zap.String("pk", pk), zap.String("name", name)) From ff4a88baeccd009fb71fe3ecc74d252cfa02bd5d Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Sat, 13 Jun 2026 20:34:18 -0700 Subject: [PATCH 25/40] fix lint --- internal/providers/luma/client.go | 8 ++++---- plugins/backgrounds/checkin_complete.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go index 3a0ef56d9b..82057da4b1 100644 --- a/internal/providers/luma/client.go +++ b/internal/providers/luma/client.go @@ -4,11 +4,11 @@ package luma import ( + "bytes" "context" "encoding/json" "errors" "fmt" - "bytes" "io" "net/http" "net/url" @@ -19,9 +19,9 @@ import ( ) const ( - DefaultBaseURL = "https://public-api.luma.com" - GetGuestPath = "/v1/event/get-guest" - CheckInURL = "https://api.luma.com/event/admin/update-check-in" + DefaultBaseURL = "https://public-api.luma.com" + GetGuestPath = "/v1/event/get-guest" + CheckInURL = "https://api.luma.com/event/admin/update-check-in" ) var ( diff --git a/plugins/backgrounds/checkin_complete.go b/plugins/backgrounds/checkin_complete.go index 610b89c8d5..aa978ec281 100644 --- a/plugins/backgrounds/checkin_complete.go +++ b/plugins/backgrounds/checkin_complete.go @@ -19,12 +19,12 @@ func init() { } type checkinCompleteConfig struct { - FaceBaseURL string `json:"face_http_base_url"` - FaceRecentSec float64 `json:"face_recent_sec"` - FaceMinArea float64 `json:"min_face_area"` - PollSec float64 `json:"poll_interval_sec"` - ScanIOKey string `json:"scan_io_key"` - GracePeriodSec float64 `json:"grace_period_sec"` + FaceBaseURL string `json:"face_http_base_url"` + FaceRecentSec float64 `json:"face_recent_sec"` + FaceMinArea float64 `json:"min_face_area"` + PollSec float64 `json:"poll_interval_sec"` + ScanIOKey string `json:"scan_io_key"` + GracePeriodSec float64 `json:"grace_period_sec"` } // CheckinComplete publishes checkin_complete:true to ModeContext once a From 92e8e73ce4b43eb33bcf69f3b529f1f782f903aa Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Sat, 13 Jun 2026 20:42:28 -0700 Subject: [PATCH 26/40] fix tests --- internal/providers/luma/client_test.go | 243 +++++++++++++++++++ plugins/backgrounds/checkin_complete_test.go | 41 ++++ plugins/backgrounds/face_size_watch_test.go | 37 +++ 3 files changed, 321 insertions(+) create mode 100644 internal/providers/luma/client_test.go create mode 100644 plugins/backgrounds/checkin_complete_test.go create mode 100644 plugins/backgrounds/face_size_watch_test.go diff --git a/internal/providers/luma/client_test.go b/internal/providers/luma/client_test.go new file mode 100644 index 0000000000..ff199f417a --- /dev/null +++ b/internal/providers/luma/client_test.go @@ -0,0 +1,243 @@ +package luma + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" +) + +// roundTripFunc lets us stub HTTP responses inline. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func stubClient(fn roundTripFunc) *http.Client { + return &http.Client{Transport: fn} +} + +func newTestClient(doer HTTPDoer, opts ...func(*Client)) *Client { + c := NewClient("https://test.luma.com", "test-key", "evt-123", 5*time.Second, opts...) + c.SetHTTPDoer(doer) + return c +} + +func jsonResp(code int, body any) *http.Response { + b, _ := json.Marshal(body) + return &http.Response{ + StatusCode: code, + Body: io.NopCloser(strings.NewReader(string(b))), + Header: http.Header{"Content-Type": []string{"application/json"}}, + } +} + +func TestGetGuest_OK_Envelope(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + if !strings.Contains(req.URL.Path, GetGuestPath) { + t.Fatalf("unexpected path: %s", req.URL.Path) + } + if req.Header.Get("x-luma-api-key") != "test-key" { + t.Fatalf("missing api key header") + } + return jsonResp(200, guestEnvelope{ + Guest: &Guest{APIID: "gst-1", UserFirstName: "Alice", UserEmail: "alice@x.com"}, + Event: &struct{ APIID string `json:"api_id"` }{APIID: "evt-123"}, + }), nil + }) + + c := newTestClient(doer) + g, err := c.GetGuest(context.Background(), "g-abc") + if err != nil { + t.Fatal(err) + } + if g.APIID != "gst-1" { + t.Fatalf("got APIID %q", g.APIID) + } + if g.UserFirstName != "Alice" { + t.Fatalf("got first name %q", g.UserFirstName) + } + if g.EventAPIID != "evt-123" { + t.Fatalf("got event api id %q", g.EventAPIID) + } +} + +func TestGetGuest_OK_Bare(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + return jsonResp(200, Guest{APIID: "gst-2", UserName: "Bob Smith"}), nil + }) + + c := newTestClient(doer) + g, err := c.GetGuest(context.Background(), "g-abc") + if err != nil { + t.Fatal(err) + } + if g.APIID != "gst-2" { + t.Fatalf("got APIID %q", g.APIID) + } +} + +func TestGetGuest_NotFound(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + return jsonResp(404, map[string]string{"error": "not found"}), nil + }) + + c := newTestClient(doer) + _, err := c.GetGuest(context.Background(), "g-abc") + if err == nil || err.Error() != ErrNotFound.Error() { + t.Fatalf("expected ErrNotFound, got %v", err) + } +} + +func TestGetGuest_Unauthorized(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + return jsonResp(401, nil), nil + }) + + c := newTestClient(doer) + _, err := c.GetGuest(context.Background(), "g-abc") + if err == nil || err.Error() != ErrUnauthorized.Error() { + t.Fatalf("expected ErrUnauthorized, got %v", err) + } +} + +func TestGetGuest_429_Retry(t *testing.T) { + calls := 0 + doer := stubClient(func(req *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return jsonResp(429, nil), nil + } + return jsonResp(200, guestEnvelope{ + Guest: &Guest{APIID: "gst-retry"}, + }), nil + }) + + c := newTestClient(doer) + g, err := c.GetGuest(context.Background(), "g-abc") + if err != nil { + t.Fatal(err) + } + if g.APIID != "gst-retry" { + t.Fatalf("got APIID %q", g.APIID) + } + if calls != 2 { + t.Fatalf("expected 2 calls, got %d", calls) + } +} + +func TestCheckIn_OK(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + if req.URL.String() != CheckInURL { + t.Fatalf("unexpected url: %s", req.URL) + } + if !strings.Contains(req.Header.Get("Cookie"), "luma.auth-session-key=sess-abc") { + t.Fatalf("missing session cookie") + } + body, _ := io.ReadAll(req.Body) + var payload map[string]string + json.Unmarshal(body, &payload) + if payload["rsvp_api_id"] != "gst-1" { + t.Fatalf("wrong rsvp_api_id: %s", payload["rsvp_api_id"]) + } + if payload["check_in_status"] != "checked-in" { + t.Fatalf("wrong status: %s", payload["check_in_status"]) + } + return jsonResp(200, map[string]string{"ok": "true"}), nil + }) + + c := newTestClient(doer, WithSessionKey("sess-abc")) + err := c.CheckIn(context.Background(), &Guest{APIID: "gst-1"}) + if err != nil { + t.Fatal(err) + } +} + +func TestCheckIn_NoSessionKey(t *testing.T) { + c := NewClient("", "key", "evt-1", time.Second) + err := c.CheckIn(context.Background(), &Guest{APIID: "gst-1"}) + if err == nil || !strings.Contains(err.Error(), "no session key") { + t.Fatalf("expected no session key error, got %v", err) + } +} + +func TestCheckIn_NilGuest(t *testing.T) { + c := NewClient("", "key", "evt-1", time.Second, WithSessionKey("s")) + err := c.CheckIn(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "missing guest") { + t.Fatalf("expected missing guest error, got %v", err) + } +} + +func TestCheckIn_ServerError(t *testing.T) { + doer := stubClient(func(req *http.Request) (*http.Response, error) { + return jsonResp(500, map[string]string{"error": "internal"}), nil + }) + + c := newTestClient(doer, WithSessionKey("sess")) + err := c.CheckIn(context.Background(), &Guest{APIID: "gst-1"}) + if err == nil || !strings.Contains(err.Error(), "500") { + t.Fatalf("expected 500 error, got %v", err) + } +} + +func TestFirstName(t *testing.T) { + cases := []struct { + name string + guest *Guest + want string + }{ + {"nil guest", nil, "friend"}, + {"first name set", &Guest{UserFirstName: "Alice"}, "Alice"}, + {"only full name", &Guest{UserName: "Bob Smith"}, "Bob"}, + {"empty", &Guest{}, "friend"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := FirstName(tc.guest); got != tc.want { + t.Fatalf("got %q want %q", got, tc.want) + } + }) + } +} + +func TestFormatGreeting(t *testing.T) { + g := &Guest{ + UserFirstName: "Alice", + UserLastName: "Smith", + UserName: "Alice Smith", + UserEmail: "alice@x.com", + } + tmpl := "Hi {first_name} {last_name}, welcome {name}! Contact: {email}" + got := FormatGreeting(tmpl, g) + want := "Hi Alice Smith, welcome Alice Smith! Contact: alice@x.com" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestFormatGreeting_Nil(t *testing.T) { + got := FormatGreeting("Hello {first_name}!", nil) + if got != "Hello {first_name}!" { + t.Fatalf("got %q", got) + } +} + +func TestNewClient_Defaults(t *testing.T) { + c := NewClient("", "key", "evt-1", 0) + if c.baseURL != DefaultBaseURL { + t.Fatalf("got base url %q", c.baseURL) + } + if c.timeout != 5*time.Second { + t.Fatalf("got timeout %v", c.timeout) + } +} + +func TestEventAPIID(t *testing.T) { + c := NewClient("", "key", "evt-42", time.Second) + if c.EventAPIID() != "evt-42" { + t.Fatalf("got %q", c.EventAPIID()) + } +} diff --git a/plugins/backgrounds/checkin_complete_test.go b/plugins/backgrounds/checkin_complete_test.go new file mode 100644 index 0000000000..688fe53207 --- /dev/null +++ b/plugins/backgrounds/checkin_complete_test.go @@ -0,0 +1,41 @@ +package backgrounds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewCheckinComplete_Defaults(t *testing.T) { + bg, err := NewCheckinComplete(map[string]any{}) + require.NoError(t, err) + require.NotNil(t, bg) + + cc := bg.(*CheckinComplete) + require.Equal(t, float64(3000), cc.minArea) + require.Equal(t, "QRScannerRTSP", cc.scanIOKey) + require.False(t, cc.scanSeen) + require.NotNil(t, cc.face) + require.NotNil(t, cc.log) +} + +func TestNewCheckinComplete_CustomConfig(t *testing.T) { + bg, err := NewCheckinComplete(map[string]any{ + "face_http_base_url": "http://localhost:9999", + "min_face_area": 8000.0, + "poll_interval_sec": 2.0, + "scan_io_key": "QRScanner", + "grace_period_sec": 10.0, + }) + require.NoError(t, err) + + cc := bg.(*CheckinComplete) + require.Equal(t, float64(8000), cc.minArea) + require.Equal(t, "QRScanner", cc.scanIOKey) +} + +func TestCheckinComplete_Stop(t *testing.T) { + bg, err := NewCheckinComplete(map[string]any{}) + require.NoError(t, err) + bg.(*CheckinComplete).Stop() +} diff --git a/plugins/backgrounds/face_size_watch_test.go b/plugins/backgrounds/face_size_watch_test.go new file mode 100644 index 0000000000..7d08979081 --- /dev/null +++ b/plugins/backgrounds/face_size_watch_test.go @@ -0,0 +1,37 @@ +package backgrounds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewFaceSizeWatch_Defaults(t *testing.T) { + bg, err := NewFaceSizeWatch(map[string]any{}) + require.NoError(t, err) + require.NotNil(t, bg) + + fsw := bg.(*FaceSizeWatch) + require.Equal(t, float64(3000), fsw.minArea) + require.NotNil(t, fsw.provider) + require.NotNil(t, fsw.log) +} + +func TestNewFaceSizeWatch_CustomConfig(t *testing.T) { + bg, err := NewFaceSizeWatch(map[string]any{ + "face_http_base_url": "http://localhost:9999", + "face_recent_sec": 2.0, + "face_poll_interval_sec": 1.0, + "min_face_area": 5000.0, + }) + require.NoError(t, err) + + fsw := bg.(*FaceSizeWatch) + require.Equal(t, float64(5000), fsw.minArea) +} + +func TestFaceSizeWatch_Stop(t *testing.T) { + bg, err := NewFaceSizeWatch(map[string]any{}) + require.NoError(t, err) + bg.(*FaceSizeWatch).Stop() +} From 512cfba231e4463522141a48dbb666976f66fc79 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Sat, 13 Jun 2026 20:44:03 -0700 Subject: [PATCH 27/40] fix lint --- internal/providers/luma/client_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/providers/luma/client_test.go b/internal/providers/luma/client_test.go index ff199f417a..3280c7cfe6 100644 --- a/internal/providers/luma/client_test.go +++ b/internal/providers/luma/client_test.go @@ -44,7 +44,9 @@ func TestGetGuest_OK_Envelope(t *testing.T) { } return jsonResp(200, guestEnvelope{ Guest: &Guest{APIID: "gst-1", UserFirstName: "Alice", UserEmail: "alice@x.com"}, - Event: &struct{ APIID string `json:"api_id"` }{APIID: "evt-123"}, + Event: &struct { + APIID string `json:"api_id"` + }{APIID: "evt-123"}, }), nil }) From 7b1e1d9a52bf435fb4adce71b29827b7afffc381 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Sat, 13 Jun 2026 20:50:52 -0700 Subject: [PATCH 28/40] fix lint --- internal/providers/luma/client.go | 4 ++-- internal/providers/luma/client_test.go | 2 +- plugins/inputs/qr_scanner/export_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go index 82057da4b1..c18387abce 100644 --- a/internal/providers/luma/client.go +++ b/internal/providers/luma/client.go @@ -119,7 +119,7 @@ func (c *Client) GetGuest(ctx context.Context, pk string) (*Guest, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { @@ -222,7 +222,7 @@ func (c *Client) CheckIn(ctx context.Context, guest *Guest) error { if err != nil { return fmt.Errorf("luma check-in: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() respBody, _ := io.ReadAll(resp.Body) if resp.StatusCode != http.StatusOK { diff --git a/internal/providers/luma/client_test.go b/internal/providers/luma/client_test.go index 3280c7cfe6..c44a57f30e 100644 --- a/internal/providers/luma/client_test.go +++ b/internal/providers/luma/client_test.go @@ -140,7 +140,7 @@ func TestCheckIn_OK(t *testing.T) { } body, _ := io.ReadAll(req.Body) var payload map[string]string - json.Unmarshal(body, &payload) + _ = json.Unmarshal(body, &payload) if payload["rsvp_api_id"] != "gst-1" { t.Fatalf("wrong rsvp_api_id: %s", payload["rsvp_api_id"]) } diff --git a/plugins/inputs/qr_scanner/export_test.go b/plugins/inputs/qr_scanner/export_test.go index 5c53e34fc0..7cda7bec19 100644 --- a/plugins/inputs/qr_scanner/export_test.go +++ b/plugins/inputs/qr_scanner/export_test.go @@ -42,12 +42,12 @@ func NewTestSensor(lookup GuestLookup, greetingTmpl, expectedEventID string, tim type TestSensor struct{ *sensor } func (s *TestSensor) FormatScanMessage(ctx context.Context, pk, eventID string) string { - return s.sensor.formatScanMessage(ctx, pk, eventID) + return s.formatScanMessage(ctx, pk, eventID) } // SetSpeaker injects a fake ttsSpeaker so tests can verify the direct-TTS // path without touching the real ElevenLabs singleton. -func (s *TestSensor) SetSpeaker(sp Speaker) { s.sensor.speak = sp } +func (s *TestSensor) SetSpeaker(sp Speaker) { s.speak = sp } // Speaker is the test-side alias for the unexported ttsSpeaker interface. type Speaker = ttsSpeaker From 7f4b363a54ad776ecf6c3b49f5ce3286875320cf Mon Sep 17 00:00:00 2001 From: openmindev <147775420+openminddev@users.noreply.github.com> Date: Sun, 14 Jun 2026 20:18:52 -0700 Subject: [PATCH 29/40] Add Luma check-in provider and plugins Introduce Luma check-in support and refactor QR scanner into a Luma-focused flow. - Add config/luma_checkin_local.json5 with default modes and Luma/TTS settings. - New provider package internal/providers/luma: RecordCheckIn/LastCheckIn and tests. - Add hooks/luma_hook.go and tests; refactor greeting hook to extract announceGenerated for shared LLM->TTS flow. - Update internal/providers/luma client: simplify HTTP client usage and remove unused greeting formatting helpers; adjust tests. - Register and add Luma background plugin: move/rename checkin background to plugins/backgrounds/luma (LumaCheckin) and adjust logic to use luma.LastCheckIn to trigger checkin_complete; update tests. - Replace QR scanner input with plugins/inputs/luma_checkin: rename files/packages, simplify formatScanMessage to perform Luma lookup/check-in and record check-ins (luma.RecordCheckIn), remove direct TTS bypass and related helpers; update registrations (NewLumaCheckin/NewLumaCheckinRTSP) and tests accordingly. - Update plugin import registrations and prune removed test helpers and exports. Overall this change integrates Luma guest lookup/check-in into the codebase, centralizes greeting generation, and renames/refactors the QR scanner into a dedicated Luma check-in plugin. --- internal/hooks/greeting_hook.go | 22 ++- internal/hooks/luma_hook.go | 25 +++ internal/hooks/luma_hook_test.go | 19 ++ internal/providers/luma/checkin.go | 21 +++ internal/providers/luma/checkin_test.go | 31 ++++ internal/providers/luma/client.go | 38 +--- internal/providers/luma/client_test.go | 26 +-- plugins/backgrounds/backgrounds.go | 1 + .../backgrounds/{ => luma}/face_size_watch.go | 2 +- .../{ => luma}/face_size_watch_test.go | 2 +- .../luma_checkin.go} | 45 ++--- .../luma_checkin_test.go} | 17 +- plugins/inputs/inputs.go | 2 +- .../{qr_scanner => luma_checkin}/debounce.go | 2 +- .../debounce_test.go | 26 +-- .../{qr_scanner => luma_checkin}/decode.go | 4 +- .../decode_test.go | 10 +- .../luma_checkin.go} | 160 +++++----------- .../{qr_scanner => luma_checkin}/url_parse.go | 6 +- .../url_parse_test.go | 6 +- plugins/inputs/qr_scanner/export_test.go | 67 ------- .../qr_scanner/format_scan_message_test.go | 172 ------------------ 22 files changed, 206 insertions(+), 498 deletions(-) create mode 100644 internal/hooks/luma_hook.go create mode 100644 internal/hooks/luma_hook_test.go create mode 100644 internal/providers/luma/checkin.go create mode 100644 internal/providers/luma/checkin_test.go rename plugins/backgrounds/{ => luma}/face_size_watch.go (99%) rename plugins/backgrounds/{ => luma}/face_size_watch_test.go (97%) rename plugins/backgrounds/{checkin_complete.go => luma/luma_checkin.go} (68%) rename plugins/backgrounds/{checkin_complete_test.go => luma/luma_checkin_test.go} (57%) rename plugins/inputs/{qr_scanner => luma_checkin}/debounce.go (97%) rename plugins/inputs/{qr_scanner => luma_checkin}/debounce_test.go (67%) rename plugins/inputs/{qr_scanner => luma_checkin}/decode.go (90%) rename plugins/inputs/{qr_scanner => luma_checkin}/decode_test.go (87%) rename plugins/inputs/{qr_scanner/qr_scanner.go => luma_checkin/luma_checkin.go} (64%) rename plugins/inputs/{qr_scanner => luma_checkin}/url_parse.go (75%) rename plugins/inputs/{qr_scanner => luma_checkin}/url_parse_test.go (94%) delete mode 100644 plugins/inputs/qr_scanner/export_test.go delete mode 100644 plugins/inputs/qr_scanner/format_scan_message_test.go diff --git a/internal/hooks/greeting_hook.go b/internal/hooks/greeting_hook.go index 5ec725a757..4e0dd802f3 100644 --- a/internal/hooks/greeting_hook.go +++ b/internal/hooks/greeting_hook.go @@ -32,18 +32,20 @@ const defaultGreetingPrompt = "You are {robot_name}, a friendly robot greeting w "Finish by offering help, for example: \"{help_message}\". " + "Respond with only the greeting text, with no quotes or commentary." -// greetingStartHook handles the start of a greeting conversation by generating a -// greeting message using an LLM and sending it to the TTS provider. func (r *Runner) greetingStartHook(ctx context.Context, cfg, vars map[string]any) error { + return r.announceGenerated(ctx, cfg, vars, defaultGreetingPrompt, "How can I help you today?") +} + +func (r *Runner) announceGenerated(ctx context.Context, cfg, vars map[string]any, defaultPrompt, defaultHelp string) error { provider, err := r.greetingTTSProvider(cfg) if err != nil { - r.log.Error("greeting_start_hook: error", zap.Error(err)) + r.log.Error("greeting hook: error", zap.Error(err)) return err } robotName := formatTemplate(stringVal(cfg, "robot_name"), vars) - helpMessage := "How can I help you today?" + helpMessage := defaultHelp if custom := formatTemplate(stringVal(cfg, "custom_message"), vars); custom != "" { helpMessage = custom } @@ -51,20 +53,20 @@ func (r *Runner) greetingStartHook(ctx context.Context, cfg, vars map[string]any face := providers.NewFacePresenceProvider(providers.FacePresenceConfig{}) snapshot, snapErr := face.FetchSnapshot(ctx) if snapErr != nil { - r.log.Warn("greeting_start_hook: face snapshot failed", zap.Error(snapErr)) + r.log.Warn("greeting hook: face snapshot failed", zap.Error(snapErr)) } memContext := r.recallMemory(ctx, snapshot.ClosestUUID) - if greeting, genErr := r.generateGreeting(ctx, cfg, vars, snapshot, memContext, robotName, helpMessage); genErr != nil { - r.log.Warn("greeting_start_hook: llm generation failed, using static greeting", zap.Error(genErr)) + if greeting, genErr := r.generateGreeting(ctx, cfg, vars, snapshot, memContext, robotName, helpMessage, defaultPrompt); genErr != nil { + r.log.Warn("greeting hook: llm generation failed, using static greeting", zap.Error(genErr)) provider.AddText(staticGreeting(snapshot, snapErr, robotName, helpMessage)) } else { r.log.Info("greeting generated successfully", zap.String("greeting", greeting)) provider.AddText(greeting) } - r.log.Info("greeting start hook executed successfully") + r.log.Info("greeting hook executed successfully") return nil } @@ -92,7 +94,7 @@ func memoryClause(memContext string) string { } // generateGreeting constructs a prompt using the snapshot and other context, calls the LLM to generate a greeting, and returns the greeting text. -func (r *Runner) generateGreeting(ctx context.Context, cfg, vars map[string]any, snapshot providers.PresenceSnapshot, memContext, robotName, helpMessage string) (string, error) { +func (r *Runner) generateGreeting(ctx context.Context, cfg, vars map[string]any, snapshot providers.PresenceSnapshot, memContext, robotName, helpMessage, defaultPrompt string) (string, error) { if robotName == "" { robotName = "a friendly robot" } @@ -106,7 +108,7 @@ func (r *Runner) generateGreeting(ctx context.Context, cfg, vars map[string]any, promptTemplate := stringVal(cfg, "prompt") if strings.TrimSpace(promptTemplate) == "" { - promptTemplate = defaultGreetingPrompt + promptTemplate = defaultPrompt } promptVars := make(map[string]any, len(vars)+6) diff --git a/internal/hooks/luma_hook.go b/internal/hooks/luma_hook.go new file mode 100644 index 0000000000..1085d925e0 --- /dev/null +++ b/internal/hooks/luma_hook.go @@ -0,0 +1,25 @@ +package hooks + +import ( + "context" +) + +func init() { + RegisterHook("luma_hook", "luma_intro_hook", (*Runner).lumaIntroHook) +} + +const defaultLumaHelp = "Please show me your Luma QR code to check in. If you didn't register on Luma, just tell me your name." + +const defaultLumaPrompt = "You are {robot_name}, a friendly robot welcoming a guest to an event. " + + "The current time is {current_time}. " + + "Generate a single warm, natural spoken greeting of one or two short sentences. " + + "Here is what you currently see: {scene}. " + + "Make it feel personal and present by naturally referencing something specific from what you see; never invent anything. " + + "If a specific person is recognized ({closest_name}), greet them by name; otherwise greet generically. " + + "{memory}" + + "Finish by asking the guest to show their Luma QR code so you can check them in, and tell them that if they did not register on Luma they can simply say their name instead, for example: \"{help_message}\". " + + "Respond with only the greeting text, with no quotes or commentary." + +func (r *Runner) lumaIntroHook(ctx context.Context, cfg, vars map[string]any) error { + return r.announceGenerated(ctx, cfg, vars, defaultLumaPrompt, defaultLumaHelp) +} diff --git a/internal/hooks/luma_hook_test.go b/internal/hooks/luma_hook_test.go new file mode 100644 index 0000000000..a273449033 --- /dev/null +++ b/internal/hooks/luma_hook_test.go @@ -0,0 +1,19 @@ +package hooks + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLumaHookRegistered(t *testing.T) { + _, ok := lookupHook("luma_hook", "luma_intro_hook") + require.True(t, ok) +} + +func TestLumaPromptMentionsLuma(t *testing.T) { + require.True(t, strings.Contains(defaultLumaPrompt, "{help_message}"), + "prompt must interpolate the help message") + require.Contains(t, strings.ToLower(defaultLumaHelp), "luma") +} diff --git a/internal/providers/luma/checkin.go b/internal/providers/luma/checkin.go new file mode 100644 index 0000000000..be97d8144e --- /dev/null +++ b/internal/providers/luma/checkin.go @@ -0,0 +1,21 @@ +package luma + +import ( + "sync/atomic" + "time" +) + +type CheckIn struct { + Name string + Time time.Time +} + +var lastCheckIn atomic.Pointer[CheckIn] + +func RecordCheckIn(name string, t time.Time) { + lastCheckIn.Store(&CheckIn{Name: name, Time: t}) +} + +func LastCheckIn() *CheckIn { + return lastCheckIn.Load() +} diff --git a/internal/providers/luma/checkin_test.go b/internal/providers/luma/checkin_test.go new file mode 100644 index 0000000000..028e338fe8 --- /dev/null +++ b/internal/providers/luma/checkin_test.go @@ -0,0 +1,31 @@ +package luma + +import ( + "testing" + "time" +) + +func TestRecordAndLastCheckIn(t *testing.T) { + if got := LastCheckIn(); got != nil { + t.Fatalf("expected nil before any publish, got %+v", got) + } + + t1 := time.Unix(1_700_000_000, 0) + RecordCheckIn("Ada", t1) + + got := LastCheckIn() + if got == nil { + t.Fatal("expected a check-in after publish") + } + if got.Name != "Ada" || !got.Time.Equal(t1) { + t.Fatalf("unexpected check-in: %+v", got) + } + + // A later publish replaces the previous value. + t2 := t1.Add(time.Minute) + RecordCheckIn("Grace", t2) + got = LastCheckIn() + if got.Name != "Grace" || !got.Time.Equal(t2) { + t.Fatalf("expected latest check-in to be Grace@t2, got %+v", got) + } +} diff --git a/internal/providers/luma/client.go b/internal/providers/luma/client.go index c18387abce..904562aff5 100644 --- a/internal/providers/luma/client.go +++ b/internal/providers/luma/client.go @@ -1,6 +1,3 @@ -// Package luma provides a thin client for Luma's public API. Currently it only -// exposes /v1/event/get-guest, which is enough to look up a guest by the pk -// embedded in a check-in QR code and surface their name to the rest of OM1. package luma import ( @@ -53,17 +50,12 @@ type guestEnvelope struct { } `json:"event"` } -// HTTPDoer is satisfied by *http.Client; abstracted for tests. -type HTTPDoer interface { - Do(req *http.Request) (*http.Response, error) -} - type Client struct { baseURL string apiKey string eventAPIID string sessionKey string - http HTTPDoer + http *http.Client timeout time.Duration } @@ -91,15 +83,10 @@ func WithSessionKey(key string) func(*Client) { return func(c *Client) { c.sessionKey = key } } -// SetHTTPDoer overrides the underlying http client. Intended for tests. -func (c *Client) SetHTTPDoer(d HTTPDoer) { c.http = d } - // EventAPIID returns the configured event id. func (c *Client) EventAPIID() string { return c.eventAPIID } -// GetGuest fetches the guest record for the given key (g- or ticket key). -// Luma's get-guest requires both event_id (the configured event) and id (the -// guest/ticket key from the QR). +// GetGuest retrieves a guest record by its API ID. func (c *Client) GetGuest(ctx context.Context, pk string) (*Guest, error) { ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() @@ -186,8 +173,6 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { return c.http.Do(req) } -// CheckIn marks a guest as checked-in via Luma's admin endpoint. -// Requires a session key (from a logged-in browser session). func (c *Client) CheckIn(ctx context.Context, guest *Guest) error { if c.sessionKey == "" { return fmt.Errorf("luma check-in: no session key configured") @@ -247,22 +232,3 @@ func FirstName(g *Guest) string { } return "friend" } - -// FormatGreeting interpolates {first_name}, {last_name}, {name}, {email} into -// template using the guest's fields. -func FormatGreeting(template string, g *Guest) string { - if g == nil { - return template - } - name := g.UserName - if name == "" { - name = FirstName(g) - } - r := strings.NewReplacer( - "{first_name}", FirstName(g), - "{last_name}", g.UserLastName, - "{name}", name, - "{email}", g.UserEmail, - ) - return r.Replace(template) -} diff --git a/internal/providers/luma/client_test.go b/internal/providers/luma/client_test.go index c44a57f30e..72983641a0 100644 --- a/internal/providers/luma/client_test.go +++ b/internal/providers/luma/client_test.go @@ -19,9 +19,9 @@ func stubClient(fn roundTripFunc) *http.Client { return &http.Client{Transport: fn} } -func newTestClient(doer HTTPDoer, opts ...func(*Client)) *Client { +func newTestClient(doer *http.Client, opts ...func(*Client)) *Client { c := NewClient("https://test.luma.com", "test-key", "evt-123", 5*time.Second, opts...) - c.SetHTTPDoer(doer) + c.http = doer return c } @@ -205,28 +205,6 @@ func TestFirstName(t *testing.T) { } } -func TestFormatGreeting(t *testing.T) { - g := &Guest{ - UserFirstName: "Alice", - UserLastName: "Smith", - UserName: "Alice Smith", - UserEmail: "alice@x.com", - } - tmpl := "Hi {first_name} {last_name}, welcome {name}! Contact: {email}" - got := FormatGreeting(tmpl, g) - want := "Hi Alice Smith, welcome Alice Smith! Contact: alice@x.com" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatGreeting_Nil(t *testing.T) { - got := FormatGreeting("Hello {first_name}!", nil) - if got != "Hello {first_name}!" { - t.Fatalf("got %q", got) - } -} - func TestNewClient_Defaults(t *testing.T) { c := NewClient("", "key", "evt-1", 0) if c.baseURL != DefaultBaseURL { diff --git a/plugins/backgrounds/backgrounds.go b/plugins/backgrounds/backgrounds.go index d20c9ed521..28300131a6 100644 --- a/plugins/backgrounds/backgrounds.go +++ b/plugins/backgrounds/backgrounds.go @@ -1,6 +1,7 @@ package backgrounds import ( + _ "github.com/openmind/om1/plugins/backgrounds/luma" _ "github.com/openmind/om1/plugins/backgrounds/unitree/go2" _ "github.com/openmind/om1/plugins/backgrounds/vlm" ) diff --git a/plugins/backgrounds/face_size_watch.go b/plugins/backgrounds/luma/face_size_watch.go similarity index 99% rename from plugins/backgrounds/face_size_watch.go rename to plugins/backgrounds/luma/face_size_watch.go index 92c87c7f96..321a45e3a0 100644 --- a/plugins/backgrounds/face_size_watch.go +++ b/plugins/backgrounds/luma/face_size_watch.go @@ -1,4 +1,4 @@ -package backgrounds +package luma import ( "context" diff --git a/plugins/backgrounds/face_size_watch_test.go b/plugins/backgrounds/luma/face_size_watch_test.go similarity index 97% rename from plugins/backgrounds/face_size_watch_test.go rename to plugins/backgrounds/luma/face_size_watch_test.go index 7d08979081..a6f5ff5961 100644 --- a/plugins/backgrounds/face_size_watch_test.go +++ b/plugins/backgrounds/luma/face_size_watch_test.go @@ -1,4 +1,4 @@ -package backgrounds +package luma import ( "testing" diff --git a/plugins/backgrounds/checkin_complete.go b/plugins/backgrounds/luma/luma_checkin.go similarity index 68% rename from plugins/backgrounds/checkin_complete.go rename to plugins/backgrounds/luma/luma_checkin.go index aa978ec281..26748c5632 100644 --- a/plugins/backgrounds/checkin_complete.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -1,9 +1,8 @@ -package backgrounds +package luma import ( "context" "encoding/json" - "strings" "time" "go.uber.org/zap" @@ -11,11 +10,12 @@ import ( bg "github.com/openmind/om1/internal/backgrounds" "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" + "github.com/openmind/om1/internal/providers/luma" "github.com/openmind/om1/internal/util" ) func init() { - bg.Register("CheckinComplete", NewCheckinComplete) + bg.Register("LumaCheckin", LumaCheckin) } type checkinCompleteConfig struct { @@ -23,25 +23,19 @@ type checkinCompleteConfig struct { FaceRecentSec float64 `json:"face_recent_sec"` FaceMinArea float64 `json:"min_face_area"` PollSec float64 `json:"poll_interval_sec"` - ScanIOKey string `json:"scan_io_key"` GracePeriodSec float64 `json:"grace_period_sec"` } -// CheckinComplete publishes checkin_complete:true to ModeContext once a -// successful QR scan has been recorded AND the guest's face is no longer -// visible (or too small). type CheckinComplete struct { log *zap.Logger face *providers.FacePresenceProvider period time.Duration - scanIOKey string gracePeriod time.Duration minArea float64 - scanSeen bool - scanTime time.Time + lastHandled time.Time // timestamp of the scan we've already acted on } -func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { +func LumaCheckin(configMap map[string]any) (bg.Background, error) { var cfg checkinCompleteConfig if b, err := json.Marshal(configMap); err == nil { _ = json.Unmarshal(b, &cfg) @@ -58,9 +52,6 @@ func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { if cfg.PollSec <= 0 { cfg.PollSec = 1.0 } - if cfg.ScanIOKey == "" { - cfg.ScanIOKey = "QRScannerRTSP" - } if cfg.GracePeriodSec <= 0 { cfg.GracePeriodSec = 5.0 } @@ -74,7 +65,6 @@ func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { }) log.Info("initialized", - zap.String("scan_io_key", cfg.ScanIOKey), zap.Float64("grace_period_sec", cfg.GracePeriodSec), ) @@ -82,32 +72,23 @@ func NewCheckinComplete(configMap map[string]any) (bg.Background, error) { log: log, face: face, period: time.Duration(cfg.PollSec * float64(time.Second)), - scanIOKey: cfg.ScanIOKey, gracePeriod: time.Duration(cfg.GracePeriodSec * float64(time.Second)), minArea: cfg.FaceMinArea, }, nil } func (c *CheckinComplete) Run(ctx context.Context) { - // Step 1: check if a successful scan has been recorded. - if !c.scanSeen { - if in := providers.IO().GetInput(c.scanIOKey); in != nil { - if strings.HasPrefix(in.Input, "qr_scan: name=") { - c.scanSeen = true - c.scanTime = in.Timestamp - c.log.Info("successful scan detected", zap.String("value", in.Input)) - } - } - } - - if !c.scanSeen { + // Step 1: pick up the latest successful check-in. Skip when there is none yet + // or when we've already acted on it. + checkin := luma.LastCheckIn() + if checkin == nil || !checkin.Time.After(c.lastHandled) { util.Sleep(ctx, c.period) return } - // Step 2: wait a grace period after the scan before checking for face + // Step 2: wait a grace period after the check-in before checking for face // departure, so the greeting has time to play. - if time.Since(c.scanTime) < c.gracePeriod { + if time.Since(checkin.Time) < c.gracePeriod { util.Sleep(ctx, c.period) return } @@ -133,9 +114,9 @@ func (c *CheckinComplete) Run(ctx context.Context) { return } - c.log.Info("guest departed after successful scan, triggering transition") + c.log.Info("guest departed after successful check-in, triggering transition", zap.String("name", checkin.Name)) providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) - c.scanSeen = false + c.lastHandled = checkin.Time util.Sleep(ctx, c.period) } diff --git a/plugins/backgrounds/checkin_complete_test.go b/plugins/backgrounds/luma/luma_checkin_test.go similarity index 57% rename from plugins/backgrounds/checkin_complete_test.go rename to plugins/backgrounds/luma/luma_checkin_test.go index 688fe53207..628a06abb7 100644 --- a/plugins/backgrounds/checkin_complete_test.go +++ b/plugins/backgrounds/luma/luma_checkin_test.go @@ -1,4 +1,4 @@ -package backgrounds +package luma import ( "testing" @@ -6,36 +6,33 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewCheckinComplete_Defaults(t *testing.T) { - bg, err := NewCheckinComplete(map[string]any{}) +func TestLumaCheckin_Defaults(t *testing.T) { + bg, err := LumaCheckin(map[string]any{}) require.NoError(t, err) require.NotNil(t, bg) cc := bg.(*CheckinComplete) require.Equal(t, float64(3000), cc.minArea) - require.Equal(t, "QRScannerRTSP", cc.scanIOKey) - require.False(t, cc.scanSeen) + require.True(t, cc.lastHandled.IsZero()) require.NotNil(t, cc.face) require.NotNil(t, cc.log) } -func TestNewCheckinComplete_CustomConfig(t *testing.T) { - bg, err := NewCheckinComplete(map[string]any{ +func TestLumaCheckin_CustomConfig(t *testing.T) { + bg, err := LumaCheckin(map[string]any{ "face_http_base_url": "http://localhost:9999", "min_face_area": 8000.0, "poll_interval_sec": 2.0, - "scan_io_key": "QRScanner", "grace_period_sec": 10.0, }) require.NoError(t, err) cc := bg.(*CheckinComplete) require.Equal(t, float64(8000), cc.minArea) - require.Equal(t, "QRScanner", cc.scanIOKey) } func TestCheckinComplete_Stop(t *testing.T) { - bg, err := NewCheckinComplete(map[string]any{}) + bg, err := LumaCheckin(map[string]any{}) require.NoError(t, err) bg.(*CheckinComplete).Stop() } diff --git a/plugins/inputs/inputs.go b/plugins/inputs/inputs.go index 818000c1cd..990308b657 100644 --- a/plugins/inputs/inputs.go +++ b/plugins/inputs/inputs.go @@ -2,7 +2,7 @@ package inputs import ( _ "github.com/openmind/om1/plugins/inputs/asr" - _ "github.com/openmind/om1/plugins/inputs/qr_scanner" + _ "github.com/openmind/om1/plugins/inputs/luma_checkin" _ "github.com/openmind/om1/plugins/inputs/unitree/go2" _ "github.com/openmind/om1/plugins/inputs/vlm" ) diff --git a/plugins/inputs/qr_scanner/debounce.go b/plugins/inputs/luma_checkin/debounce.go similarity index 97% rename from plugins/inputs/qr_scanner/debounce.go rename to plugins/inputs/luma_checkin/debounce.go index 84901c1075..6b0c3c2a57 100644 --- a/plugins/inputs/qr_scanner/debounce.go +++ b/plugins/inputs/luma_checkin/debounce.go @@ -1,4 +1,4 @@ -package qr_scanner +package luma_checkin import ( "sync" diff --git a/plugins/inputs/qr_scanner/debounce_test.go b/plugins/inputs/luma_checkin/debounce_test.go similarity index 67% rename from plugins/inputs/qr_scanner/debounce_test.go rename to plugins/inputs/luma_checkin/debounce_test.go index 52e424bf5d..17ebcb4460 100644 --- a/plugins/inputs/qr_scanner/debounce_test.go +++ b/plugins/inputs/luma_checkin/debounce_test.go @@ -1,16 +1,14 @@ -package qr_scanner_test +package luma_checkin import ( "testing" "time" - - "github.com/openmind/om1/plugins/inputs/qr_scanner" ) func TestDebouncerAcceptsThenRejects(t *testing.T) { - d := qr_scanner.NewDebouncer(30 * time.Second) + d := newDebouncer(30 * time.Second) now := time.Unix(1_700_000_000, 0) - d.SetNow(func() time.Time { return now }) + d.now = func() time.Time { return now } if !d.TryRecord("g-1") { t.Fatalf("first record should be accepted") @@ -24,14 +22,14 @@ func TestDebouncerAcceptsThenRejects(t *testing.T) { t.Fatalf("re-record within window should be rejected") } - now = now.Add(2 * time.Second) // total 31s after first + now = now.Add(2 * time.Second) if !d.TryRecord("g-1") { t.Fatalf("re-record after window should be accepted") } } func TestDebouncerDistinctKeysIndependent(t *testing.T) { - d := qr_scanner.NewDebouncer(30 * time.Second) + d := newDebouncer(30 * time.Second) if !d.TryRecord("g-1") { t.Fatalf("g-1 first should be accepted") } @@ -41,22 +39,24 @@ func TestDebouncerDistinctKeysIndependent(t *testing.T) { } func TestDebouncerPrunesOldEntries(t *testing.T) { - d := qr_scanner.NewDebouncer(1 * time.Second) + d := newDebouncer(1 * time.Second) now := time.Unix(1_700_000_000, 0) - d.SetNow(func() time.Time { return now }) + d.now = func() time.Time { return now } + + has := func(key string) bool { _, ok := d.seen[key]; return ok } d.TryRecord("g-old") - if !d.Has("g-old") { + if !has("g-old") { t.Fatalf("expected g-old to be recorded") } - now = now.Add(15 * time.Second) // 15x window + now = now.Add(15 * time.Second) d.TryRecord("g-fresh") - if d.Has("g-old") { + if has("g-old") { t.Fatalf("expected g-old to be pruned after 15x window") } - if !d.Has("g-fresh") { + if !has("g-fresh") { t.Fatalf("expected g-fresh to be present") } } diff --git a/plugins/inputs/qr_scanner/decode.go b/plugins/inputs/luma_checkin/decode.go similarity index 90% rename from plugins/inputs/qr_scanner/decode.go rename to plugins/inputs/luma_checkin/decode.go index 446e8d81e7..4614fd9160 100644 --- a/plugins/inputs/qr_scanner/decode.go +++ b/plugins/inputs/luma_checkin/decode.go @@ -1,4 +1,4 @@ -package qr_scanner +package luma_checkin import ( "bytes" @@ -12,7 +12,7 @@ import ( // errQRNotFound signals that no QR code was found in the frame. Callers should // treat this as the common case and skip silently. -var errQRNotFound = errors.New("qr_scanner: no qr code in frame") +var errQRNotFound = errors.New("luma_checkin: no qr code in frame") // decodeQR decodes a single QR code from a JPEG-encoded frame. It returns the // raw text payload of the code, or errQRNotFound if no code is present. diff --git a/plugins/inputs/qr_scanner/decode_test.go b/plugins/inputs/luma_checkin/decode_test.go similarity index 87% rename from plugins/inputs/qr_scanner/decode_test.go rename to plugins/inputs/luma_checkin/decode_test.go index d1f937c213..cb9ccfb2fe 100644 --- a/plugins/inputs/qr_scanner/decode_test.go +++ b/plugins/inputs/luma_checkin/decode_test.go @@ -1,4 +1,4 @@ -package qr_scanner_test +package luma_checkin import ( "bytes" @@ -10,8 +10,6 @@ import ( "github.com/makiuchi-d/gozxing" "github.com/makiuchi-d/gozxing/qrcode" - - "github.com/openmind/om1/plugins/inputs/qr_scanner" ) func TestDecodeQRRoundTrip(t *testing.T) { @@ -19,7 +17,7 @@ func TestDecodeQRRoundTrip(t *testing.T) { jpegBytes := encodeQRAsJPEG(t, payload, 360) - got, err := qr_scanner.DecodeQR(jpegBytes) + got, err := decodeQR(jpegBytes) if err != nil { t.Fatalf("DecodeQR: %v", err) } @@ -31,8 +29,8 @@ func TestDecodeQRRoundTrip(t *testing.T) { func TestDecodeQRReturnsNotFoundOnBlankFrame(t *testing.T) { jpegBytes := encodeBlankJPEG(t, 100, 100) - _, err := qr_scanner.DecodeQR(jpegBytes) - if !errors.Is(err, qr_scanner.ErrQRNotFound) { + _, err := decodeQR(jpegBytes) + if !errors.Is(err, errQRNotFound) { t.Fatalf("expected ErrQRNotFound, got %v", err) } } diff --git a/plugins/inputs/qr_scanner/qr_scanner.go b/plugins/inputs/luma_checkin/luma_checkin.go similarity index 64% rename from plugins/inputs/qr_scanner/qr_scanner.go rename to plugins/inputs/luma_checkin/luma_checkin.go index 1be204c0b7..3f42a30b6d 100644 --- a/plugins/inputs/qr_scanner/qr_scanner.go +++ b/plugins/inputs/luma_checkin/luma_checkin.go @@ -1,11 +1,10 @@ -package qr_scanner +package luma_checkin import ( "context" "encoding/json" "errors" "fmt" - "strings" "sync" "time" @@ -15,64 +14,43 @@ import ( "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" "github.com/openmind/om1/internal/providers/luma" - "github.com/openmind/om1/internal/providers/tts" video "github.com/openmind/om1/internal/providers/vlm" ) const ( - scannerName = "QRScanner" - scannerRTSPName = "QRScannerRTSP" - scannerDescriptor = "QR Scanner" + scannerName = "LumaCheckin" + scannerRTSPName = "LumaCheckinRTSP" + scannerDescriptor = "Luma Check-In" scannerMaxMessages = 8 scanChannelBuffer = 4 - defaultGreetingTemplate = "Welcome, {first_name}!" - defaultLumaTimeout = 4 * time.Second + defaultLumaTimeout = 4 * time.Second ) func init() { - inputs.Register(scannerName, NewQRScanner) - inputs.Register(scannerRTSPName, NewQRScannerRTSP) + inputs.Register(scannerName, NewLumaCheckin) + inputs.Register(scannerRTSPName, NewLumaCheckinRTSP) } -// LumaConfig configures an optional guest lookup against Luma. When set, every -// scan that passes dedupe is enriched with the guest's name and a personalized -// greeting before being emitted, so downstream LLM/speak doesn't have to -// hallucinate the name. type LumaConfig struct { APIKey string `json:"api_key"` BaseURL string `json:"base_url"` EventAPIID string `json:"event_api_id"` - GreetingTemplate string `json:"greeting_template"` RequestTimeoutSeconds float64 `json:"request_timeout_seconds"` SessionKey string `json:"session_key"` } -// SpeakConfig, when set, makes the scanner push the resolved greeting straight -// to the ElevenLabs TTS provider on a successful Luma lookup, bypassing the -// LLM entirely. This avoids cortex-tick hallucinations where Gemini invents -// greetings on empty observations. -type SpeakConfig struct { - APIKey string `json:"api_key"` - ElevenLabsAPIKey string `json:"elevenlabs_api_key"` - VoiceID string `json:"voice_id"` - ModelID string `json:"model_id"` - OutputFormat string `json:"output_format"` - Rate int `json:"rate"` -} - -// Config holds the JSON configuration for the QRScanner input plugin. +// Config holds the JSON configuration for the LumaCheckin input plugin. type Config struct { - CameraIndex int `json:"camera_index"` - RTSPURL string `json:"rtsp_url"` - CaptureFPS int `json:"capture_fps"` - DecodeFPS int `json:"decode_fps"` - Width int `json:"resolution_width"` - Height int `json:"resolution_height"` - JPEGQuality int `json:"jpeg_quality"` - DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` - Luma *LumaConfig `json:"luma"` - Speak *SpeakConfig `json:"speak"` + CameraIndex int `json:"camera_index"` + RTSPURL string `json:"rtsp_url"` + CaptureFPS int `json:"capture_fps"` + DecodeFPS int `json:"decode_fps"` + Width int `json:"resolution_width"` + Height int `json:"resolution_height"` + JPEGQuality int `json:"jpeg_quality"` + DedupeWindowSeconds float64 `json:"dedupe_window_seconds"` + Luma *LumaConfig `json:"luma"` } type frameSource interface { @@ -85,12 +63,6 @@ type guestLookup interface { CheckIn(ctx context.Context, guest *luma.Guest) error } -// ttsSpeaker is the slice of *tts.ElevenLabsProvider the scanner uses to -// bypass the LLM and push a greeting straight to audio. Abstracted for tests. -type ttsSpeaker interface { - AddText(text string) -} - type sensor struct { name string cfg Config @@ -99,10 +71,8 @@ type sensor struct { debouncer *debouncer luma guestLookup - greetingTmpl string lumaTimeout time.Duration expectedEventID string - speak ttsSpeaker mu sync.Mutex messages []inputs.Message @@ -110,10 +80,20 @@ type sensor struct { cancel context.CancelFunc } -// NewQRScanner constructs a camera-backed QRScanner sensor. -func NewQRScanner(configMap map[string]any) (inputs.Sensor, error) { +// NewLumaCheckin constructs a camera-backed Luma check-in sensor. +func NewLumaCheckin(configMap map[string]any) (inputs.Sensor, error) { cfg := parseConfig(configMap) - log := newLogger(scannerName, cfg) + + log := logger.Get().Named(scannerName) + log.Info("initializing", + zap.Int("camera_index", cfg.CameraIndex), + zap.String("rtsp_url", cfg.RTSPURL), + zap.Int("capture_fps", cfg.CaptureFPS), + zap.Int("decode_fps", cfg.DecodeFPS), + zap.Int("width", cfg.Width), + zap.Int("height", cfg.Height), + zap.Float64("dedupe_window_seconds", cfg.DedupeWindowSeconds), + ) source := video.NewVideoStream(video.VideoStreamConfig{ DeviceIndex: cfg.CameraIndex, @@ -125,26 +105,14 @@ func NewQRScanner(configMap map[string]any) (inputs.Sensor, error) { return newSensor(cfg, log, source), nil } -// NewQRScannerRTSP constructs an RTSP-backed QRScanner sensor. -func NewQRScannerRTSP(configMap map[string]any) (inputs.Sensor, error) { +// NewLumaCheckinRTSP constructs an RTSP-backed Luma check-in sensor. +func NewLumaCheckinRTSP(configMap map[string]any) (inputs.Sensor, error) { cfg := parseConfig(configMap) if cfg.RTSPURL == "" { cfg.RTSPURL = "rtsp://localhost:8554/top_camera_raw" } - log := newLogger(scannerRTSPName, cfg) - - source := video.NewVideoRTSPStream(video.VideoRTSPStreamConfig{ - RTSPURL: cfg.RTSPURL, - FPS: cfg.CaptureFPS, - Width: cfg.Width, - Height: cfg.Height, - JPEGQuality: cfg.JPEGQuality, - }) - return newSensor(cfg, log, source), nil -} -func newLogger(name string, cfg Config) *zap.Logger { - log := logger.Get().Named(name) + log := logger.Get().Named(scannerRTSPName) log.Info("initializing", zap.Int("camera_index", cfg.CameraIndex), zap.String("rtsp_url", cfg.RTSPURL), @@ -154,7 +122,15 @@ func newLogger(name string, cfg Config) *zap.Logger { zap.Int("height", cfg.Height), zap.Float64("dedupe_window_seconds", cfg.DedupeWindowSeconds), ) - return log + + source := video.NewVideoRTSPStream(video.VideoRTSPStreamConfig{ + RTSPURL: cfg.RTSPURL, + FPS: cfg.CaptureFPS, + Width: cfg.Width, + Height: cfg.Height, + JPEGQuality: cfg.JPEGQuality, + }) + return newSensor(cfg, log, source), nil } func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { @@ -179,49 +155,15 @@ func newSensor(cfg Config, log *zap.Logger, source frameSource) *sensor { s.luma = luma.NewClient(cfg.Luma.BaseURL, cfg.Luma.APIKey, cfg.Luma.EventAPIID, timeout, opts...) s.lumaTimeout = timeout s.expectedEventID = cfg.Luma.EventAPIID - s.greetingTmpl = cfg.Luma.GreetingTemplate - if s.greetingTmpl == "" { - s.greetingTmpl = defaultGreetingTemplate - } log.Info("luma lookup enabled", zap.String("event_api_id", cfg.Luma.EventAPIID), zap.Duration("timeout", timeout), ) } - if cfg.Speak != nil && cfg.Speak.APIKey != "" { - s.speak = newTTSSpeaker(*cfg.Speak, log) - log.Info("direct tts enabled (greeting bypasses LLM)") - } - return s } -// newTTSSpeaker builds the singleton ElevenLabs provider with the supplied -// config. Wrapped so tests can substitute via setSpeak. -func newTTSSpeaker(cfg SpeakConfig, log *zap.Logger) ttsSpeaker { - if cfg.VoiceID == "" { - cfg.VoiceID = tts.DefaultVoiceID - } - if cfg.ModelID == "" { - cfg.ModelID = tts.DefaultModelID - } - if cfg.OutputFormat == "" { - cfg.OutputFormat = tts.DefaultOutputFormat - } - if cfg.Rate == 0 { - cfg.Rate = tts.DefaultRate - } - return tts.ElevenLabs(tts.ElevenLabsConfig{ - APIKey: cfg.APIKey, - ElevenLabsAPIKey: cfg.ElevenLabsAPIKey, - VoiceID: cfg.VoiceID, - ModelID: cfg.ModelID, - OutputFormat: cfg.OutputFormat, - Rate: cfg.Rate, - }, log.Named("speak/elevenlabs_tts")) -} - func parseConfig(configMap map[string]any) Config { var cfg Config if b, err := json.Marshal(configMap); err == nil { @@ -251,7 +193,6 @@ func parseConfig(configMap map[string]any) Config { return cfg } -// Listen starts the camera and emits scanned `pk` values as text on the returned channel. func (s *sensor) Listen(ctx context.Context) (<-chan any, error) { ctx, cancel := context.WithCancel(ctx) s.mu.Lock() @@ -383,12 +324,7 @@ func truncate(s string, n int) string { return s[:n] + "..." } -// formatScanMessage builds the observation string emitted into the LLM prompt. -// When a Luma client is configured, it does a guest lookup and embeds the -// resolved name and personalized greeting directly so the LLM doesn't have to -// hallucinate them when relaying to speak. On lookup failure or auth/event -// mismatch, the message falls back to a bare scan or a known failure marker -// so prompt logic can branch. +// formatScanMessage performs the Luma guest lookup and check-in. func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) string { if s.luma == nil { return fmt.Sprintf("qr_scan: pk=%s event=%s", pk, eventID) @@ -424,7 +360,6 @@ func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) stri } name := luma.FirstName(guest) - greeting := luma.FormatGreeting(s.greetingTmpl, guest) if err := s.luma.CheckIn(ctx, guest); err != nil { s.log.Warn("luma check-in failed", zap.String("pk", pk), zap.Error(err)) @@ -432,11 +367,8 @@ func (s *sensor) formatScanMessage(ctx context.Context, pk, eventID string) stri s.log.Info("luma check-in ok", zap.String("pk", pk), zap.String("name", name)) } - if s.speak != nil { - s.speak.AddText(greeting) - s.log.Info("greeting pushed to tts", zap.String("pk", pk), zap.String("name", name)) - } - greeting = strings.ReplaceAll(greeting, `"`, `'`) + luma.RecordCheckIn(name, time.Now()) + s.log.Info("luma lookup ok", zap.String("pk", pk), zap.String("name", name)) - return fmt.Sprintf(`qr_scan: name=%s greeting="%s"`, name, greeting) + return fmt.Sprintf("qr_scan: name=%s", name) } diff --git a/plugins/inputs/qr_scanner/url_parse.go b/plugins/inputs/luma_checkin/url_parse.go similarity index 75% rename from plugins/inputs/qr_scanner/url_parse.go rename to plugins/inputs/luma_checkin/url_parse.go index c5aabfa68e..70ef08f012 100644 --- a/plugins/inputs/qr_scanner/url_parse.go +++ b/plugins/inputs/luma_checkin/url_parse.go @@ -1,13 +1,11 @@ -package qr_scanner +package luma_checkin import ( "net/url" "strings" ) -// parseLumaCheckinURL extracts the event ID and pk from a Luma check-in URL of -// the form https://luma.com/check-in/?pk=. Accepted hosts are -// luma.com, www.luma.com, and lu.ma. +// parseLumaCheckinURL extracts the event ID and guest/ticket key from a Luma check-in URL. func parseLumaCheckinURL(s string) (eventID, pk string, ok bool) { u, err := url.Parse(strings.TrimSpace(s)) if err != nil { diff --git a/plugins/inputs/qr_scanner/url_parse_test.go b/plugins/inputs/luma_checkin/url_parse_test.go similarity index 94% rename from plugins/inputs/qr_scanner/url_parse_test.go rename to plugins/inputs/luma_checkin/url_parse_test.go index 4a15995a67..cd930a6282 100644 --- a/plugins/inputs/qr_scanner/url_parse_test.go +++ b/plugins/inputs/luma_checkin/url_parse_test.go @@ -1,9 +1,7 @@ -package qr_scanner_test +package luma_checkin import ( "testing" - - "github.com/openmind/om1/plugins/inputs/qr_scanner" ) func TestParseLumaCheckinURL(t *testing.T) { @@ -100,7 +98,7 @@ func TestParseLumaCheckinURL(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - gotEvent, gotPK, gotOK := qr_scanner.ParseLumaCheckinURL(tc.input) + gotEvent, gotPK, gotOK := parseLumaCheckinURL(tc.input) if gotOK != tc.wantOK { t.Fatalf("ok mismatch: got %v want %v", gotOK, tc.wantOK) } diff --git a/plugins/inputs/qr_scanner/export_test.go b/plugins/inputs/qr_scanner/export_test.go deleted file mode 100644 index 7cda7bec19..0000000000 --- a/plugins/inputs/qr_scanner/export_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package qr_scanner - -import ( - "context" - "time" - - "go.uber.org/zap" - - "github.com/openmind/om1/internal/providers/luma" -) - -var ( - ParseLumaCheckinURL = parseLumaCheckinURL - DecodeQR = decodeQR - ErrQRNotFound = errQRNotFound -) - -type Debouncer = debouncer - -func NewDebouncer(window time.Duration) *Debouncer { return newDebouncer(window) } - -func (d *Debouncer) SetNow(f func() time.Time) { d.now = f } -func (d *Debouncer) Has(key string) bool { _, ok := d.seen[key]; return ok } - -// GuestLookup is exported for tests so they can inject a fake without touching -// the http stack. -type GuestLookup = guestLookup - -// NewTestSensor builds a sensor wired with the given lookup, greeting template -// and expected event id. The frame source and debouncer are not used by tests -// that only exercise formatScanMessage. -func NewTestSensor(lookup GuestLookup, greetingTmpl, expectedEventID string, timeout time.Duration) *TestSensor { - return &TestSensor{&sensor{ - log: zap.NewNop(), - luma: lookup, - greetingTmpl: greetingTmpl, - lumaTimeout: timeout, - expectedEventID: expectedEventID, - }} -} - -type TestSensor struct{ *sensor } - -func (s *TestSensor) FormatScanMessage(ctx context.Context, pk, eventID string) string { - return s.formatScanMessage(ctx, pk, eventID) -} - -// SetSpeaker injects a fake ttsSpeaker so tests can verify the direct-TTS -// path without touching the real ElevenLabs singleton. -func (s *TestSensor) SetSpeaker(sp Speaker) { s.speak = sp } - -// Speaker is the test-side alias for the unexported ttsSpeaker interface. -type Speaker = ttsSpeaker - -// FakeLookup implements GuestLookup for tests. -type FakeLookup struct { - Guest *luma.Guest - Err error -} - -func (f *FakeLookup) GetGuest(_ context.Context, _ string) (*luma.Guest, error) { - return f.Guest, f.Err -} - -func (f *FakeLookup) CheckIn(_ context.Context, _ *luma.Guest) error { - return nil -} diff --git a/plugins/inputs/qr_scanner/format_scan_message_test.go b/plugins/inputs/qr_scanner/format_scan_message_test.go deleted file mode 100644 index 7bcef4bd49..0000000000 --- a/plugins/inputs/qr_scanner/format_scan_message_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package qr_scanner_test - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/openmind/om1/internal/providers/luma" - "github.com/openmind/om1/plugins/inputs/qr_scanner" -) - -func TestFormatScanMessage_NoLumaConfigured(t *testing.T) { - s := qr_scanner.NewTestSensor(nil, "", "", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := "qr_scan: pk=g-1 event=evt-abc" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_EventMismatch(t *testing.T) { - lookup := &qr_scanner.FakeLookup{} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-expected", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-other") - want := "qr_scan_failed: pk=g-1 reason=event_mismatch" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_NotFound(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Err: luma.ErrNotFound} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := "qr_scan_failed: pk=g-1 reason=guest_not_registered" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_Unauthorized(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Err: luma.ErrUnauthorized} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := "qr_scan_failed: pk=g-1 reason=luma_auth" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_GenericError(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Err: errors.New("network down")} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := "qr_scan_failed: pk=g-1 reason=lookup_error" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_EmptyResponse(t *testing.T) { - lookup := &qr_scanner.FakeLookup{} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := "qr_scan_failed: pk=g-1 reason=empty_response" - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_Success(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ - UserFirstName: "Prachi", - UserLastName: "Singh", - UserEmail: "prachi@example.com", - }} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}! Confirmed for {email}.", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := `qr_scan: name=Prachi greeting="Welcome, Prachi! Confirmed for prachi@example.com."` - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_SuccessStripsQuotesInGreeting(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ - UserFirstName: "Ada", - UserName: `Ada "the original" Lovelace`, - }} - s := qr_scanner.NewTestSensor(lookup, `Welcome, {name}!`, "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - want := `qr_scan: name=Ada greeting="Welcome, Ada 'the original' Lovelace!"` - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -func TestFormatScanMessage_EmptyEventIDSkipsMismatch(t *testing.T) { - // When the QR doesn't carry an event id, we trust the configured expected one - // and proceed with lookup rather than rejecting. - lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{UserFirstName: "Sam"}} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - got := s.FormatScanMessage(context.Background(), "g-1", "") - want := `qr_scan: name=Sam greeting="Welcome, Sam!"` - if got != want { - t.Fatalf("got %q want %q", got, want) - } -} - -// fakeSpeaker captures AddText calls so tests can assert the direct-TTS path. -type fakeSpeaker struct{ texts []string } - -func (f *fakeSpeaker) AddText(t string) { f.texts = append(f.texts, t) } - -func TestFormatScanMessage_DirectTTSOnSuccess(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{ - UserFirstName: "Prachi", - UserEmail: "prachi@example.com", - }} - speaker := &fakeSpeaker{} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}! Confirmed for {email}.", "evt-abc", time.Second) - s.SetSpeaker(speaker) - - _ = s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - - if len(speaker.texts) != 1 { - t.Fatalf("expected 1 AddText call, got %d", len(speaker.texts)) - } - want := "Welcome, Prachi! Confirmed for prachi@example.com." - if speaker.texts[0] != want { - t.Fatalf("got %q want %q", speaker.texts[0], want) - } -} - -func TestFormatScanMessage_NoSpeakOnFailure(t *testing.T) { - cases := []struct { - name string - err error - }{ - {"not_found", luma.ErrNotFound}, - {"unauthorized", luma.ErrUnauthorized}, - {"generic", errors.New("boom")}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Err: tc.err} - speaker := &fakeSpeaker{} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - s.SetSpeaker(speaker) - - _ = s.FormatScanMessage(context.Background(), "g-1", "evt-abc") - - if len(speaker.texts) != 0 { - t.Fatalf("expected no AddText calls on failure, got %v", speaker.texts) - } - }) - } -} - -func TestFormatScanMessage_NoSpeakOnEventMismatch(t *testing.T) { - lookup := &qr_scanner.FakeLookup{Guest: &luma.Guest{UserFirstName: "Sam"}} - speaker := &fakeSpeaker{} - s := qr_scanner.NewTestSensor(lookup, "Welcome, {first_name}!", "evt-abc", time.Second) - s.SetSpeaker(speaker) - - _ = s.FormatScanMessage(context.Background(), "g-1", "evt-other") - - if len(speaker.texts) != 0 { - t.Fatalf("expected no AddText on event mismatch, got %v", speaker.texts) - } -} From 6fdd21deff389933f76cdc4c123681f721bab3a0 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 15:00:32 -0700 Subject: [PATCH 30/40] add face lingering check to track checkedin attendees --- plugins/backgrounds/luma/face_size_watch.go | 19 ++- plugins/backgrounds/luma/guest_lingering.go | 146 ++++++++++++++++++++ plugins/backgrounds/luma/luma_checkin.go | 26 +++- plugins/inputs/checkin_status.go | 53 +++++++ 4 files changed, 231 insertions(+), 13 deletions(-) create mode 100644 plugins/backgrounds/luma/guest_lingering.go create mode 100644 plugins/inputs/checkin_status.go diff --git a/plugins/backgrounds/luma/face_size_watch.go b/plugins/backgrounds/luma/face_size_watch.go index 321a45e3a0..9e64dd9bb5 100644 --- a/plugins/backgrounds/luma/face_size_watch.go +++ b/plugins/backgrounds/luma/face_size_watch.go @@ -3,6 +3,7 @@ package luma import ( "context" "encoding/json" + "fmt" "time" "go.uber.org/zap" @@ -81,16 +82,22 @@ func (f *FaceSizeWatch) Run(ctx context.Context) { return } - var totalFaces int + var largestArea int + var largestTrackID int for _, face := range snap.Faces { - if float64(face.Area) >= f.minArea { - totalFaces++ + if face.Area > largestArea { + largestArea = face.Area + largestTrackID = face.TrackID } } - if totalFaces > 0 { + if float64(largestArea) >= f.minArea { f.log.Info("face close enough, triggering transition", - zap.Int("faces", totalFaces), - zap.String("closest", snap.ClosestName), + zap.Int("area", largestArea), + zap.Int("track_id", largestTrackID), + ) + providers.IO().AddInput("PrimaryGuestTrackID", + fmt.Sprintf("%d", largestTrackID), + time.Now(), ) providers.ModeContext().Publish(map[string]any{"face_close_enough": true}) } diff --git a/plugins/backgrounds/luma/guest_lingering.go b/plugins/backgrounds/luma/guest_lingering.go new file mode 100644 index 0000000000..6885a07814 --- /dev/null +++ b/plugins/backgrounds/luma/guest_lingering.go @@ -0,0 +1,146 @@ +package luma + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "go.uber.org/zap" + + bg "github.com/openmind/om1/internal/backgrounds" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers" + "github.com/openmind/om1/internal/providers/luma" + "github.com/openmind/om1/internal/util" +) + +func init() { + bg.Register("GuestLingering", NewGuestLingering) +} + +type guestLingeringConfig struct { + FaceBaseURL string `json:"face_http_base_url"` + FaceRecentSec float64 `json:"face_recent_sec"` + MinFaceArea float64 `json:"min_face_area"` + PollSec float64 `json:"poll_interval_sec"` + GracePeriodSec float64 `json:"grace_period_sec"` +} + +type GuestLingering struct { + log *zap.Logger + face *providers.FacePresenceProvider + period time.Duration + gracePeriod time.Duration + minArea float64 + lastHandled time.Time +} + +func NewGuestLingering(configMap map[string]any) (bg.Background, error) { + var cfg guestLingeringConfig + if b, err := json.Marshal(configMap); err == nil { + _ = json.Unmarshal(b, &cfg) + } + if cfg.FaceBaseURL == "" { + cfg.FaceBaseURL = "http://127.0.0.1:6793" + } + if cfg.FaceRecentSec <= 0 { + cfg.FaceRecentSec = 1.0 + } + if cfg.MinFaceArea <= 0 { + cfg.MinFaceArea = 3000 + } + if cfg.PollSec <= 0 { + cfg.PollSec = 1.0 + } + if cfg.GracePeriodSec <= 0 { + cfg.GracePeriodSec = 5.0 + } + + log := logger.Get().Named("GuestLingering") + + face := providers.NewFacePresenceProvider(providers.FacePresenceConfig{ + BaseURL: cfg.FaceBaseURL, + RecentSec: cfg.FaceRecentSec, + Timeout: 2 * time.Second, + }) + + log.Info("initialized", + zap.Float64("min_face_area", cfg.MinFaceArea), + zap.Float64("grace_period_sec", cfg.GracePeriodSec), + ) + + return &GuestLingering{ + log: log, + face: face, + period: time.Duration(cfg.PollSec * float64(time.Second)), + gracePeriod: time.Duration(cfg.GracePeriodSec * float64(time.Second)), + minArea: cfg.MinFaceArea, + }, nil +} + +func (g *GuestLingering) Run(ctx context.Context) { + checkin := luma.LastCheckIn() + if checkin == nil || !checkin.Time.After(g.lastHandled) { + util.Sleep(ctx, g.period) + return + } + + if time.Since(checkin.Time) < g.gracePeriod { + util.Sleep(ctx, g.period) + return + } + + // Read the track_id of the primary guest stored by FaceSizeWatch. + var primaryTrackID int + if in := providers.IO().GetInput("PrimaryGuestTrackID"); in != nil && in.Input != "" { + primaryTrackID, _ = strconv.Atoi(in.Input) + } + + snap, err := g.face.FetchSnapshot(ctx) + if err != nil { + if ctx.Err() == nil { + g.log.Warn("failed to fetch face snapshot", zap.Error(err)) + } + util.Sleep(ctx, g.period) + return + } + + // Check if the primary guest's face is still present. + var guestPresent bool + for _, face := range snap.Faces { + if primaryTrackID > 0 && face.TrackID == primaryTrackID && float64(face.Area) >= g.minArea { + guestPresent = true + break + } + } + // Fallback: if no track_id match (e.g. ID reassigned), use largest face. + if !guestPresent && primaryTrackID == 0 { + for _, face := range snap.Faces { + if float64(face.Area) >= g.minArea { + guestPresent = true + break + } + } + } + + if guestPresent { + g.log.Info("guest lingering after check-in", + zap.String("name", checkin.Name), + zap.Int("primary_track_id", primaryTrackID), + ) + providers.IO().AddInput("CheckinStatus", + "checkin_status: guest_lingering name="+checkin.Name, + time.Now(), + ) + } else { + providers.IO().AddInput("CheckinStatus", "", time.Now()) + g.lastHandled = checkin.Time + } + + util.Sleep(ctx, g.period) +} + +func (g *GuestLingering) Stop() { + g.log.Info("stopping") +} diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index 26748c5632..11dea0be69 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -3,6 +3,7 @@ package luma import ( "context" "encoding/json" + "strconv" "time" "go.uber.org/zap" @@ -93,7 +94,13 @@ func (c *CheckinComplete) Run(ctx context.Context) { return } - // Step 3: check if the face is gone. + // Read the primary guest's track_id stored by FaceSizeWatch. + var primaryTrackID int + if in := providers.IO().GetInput("PrimaryGuestTrackID"); in != nil && in.Input != "" { + primaryTrackID, _ = strconv.Atoi(in.Input) + } + + // Step 3: check if the primary guest's face is gone. snap, err := c.face.FetchSnapshot(ctx) if err != nil { if ctx.Err() == nil { @@ -103,15 +110,20 @@ func (c *CheckinComplete) Run(ctx context.Context) { return } - var totalFaces int for _, face := range snap.Faces { - if float64(face.Area) >= c.minArea { - totalFaces++ + if primaryTrackID > 0 && face.TrackID == primaryTrackID && float64(face.Area) >= c.minArea { + util.Sleep(ctx, c.period) + return } } - if totalFaces > 0 { - util.Sleep(ctx, c.period) - return + // Fallback: if no track_id match, check if any large face remains. + if primaryTrackID == 0 { + for _, face := range snap.Faces { + if float64(face.Area) >= c.minArea { + util.Sleep(ctx, c.period) + return + } + } } c.log.Info("guest departed after successful check-in, triggering transition", zap.String("name", checkin.Name)) diff --git a/plugins/inputs/checkin_status.go b/plugins/inputs/checkin_status.go new file mode 100644 index 0000000000..a63fe3f89d --- /dev/null +++ b/plugins/inputs/checkin_status.go @@ -0,0 +1,53 @@ +package inputs + +import ( + "context" + + "go.uber.org/zap" + + "github.com/openmind/om1/internal/inputs" + "github.com/openmind/om1/internal/logger" + "github.com/openmind/om1/internal/providers" +) + +func init() { + inputs.Register("CheckinStatus", NewCheckinStatus) +} + +type CheckinStatusSensor struct { + log *zap.Logger +} + +func NewCheckinStatus(_ map[string]any) (inputs.Sensor, error) { + log := logger.Get().Named("CheckinStatus") + log.Info("initializing") + return &CheckinStatusSensor{log: log}, nil +} + +func (s *CheckinStatusSensor) Listen(ctx context.Context) (<-chan any, error) { + out := make(chan any) + go func() { + defer close(out) + <-ctx.Done() + }() + return out, nil +} + +func (s *CheckinStatusSensor) Poll(_ context.Context) (any, error) { + return nil, nil +} + +func (s *CheckinStatusSensor) RawToText(_ context.Context, _ any) (*inputs.Message, error) { + return nil, nil +} + +func (s *CheckinStatusSensor) FormattedLatestBuffer() string { + in := providers.IO().GetInput("CheckinStatus") + if in == nil || in.Input == "" { + return "" + } + return "\n" + in.Input + "\n" +} + +func (s *CheckinStatusSensor) TriggersTick() bool { return false } +func (s *CheckinStatusSensor) Stop() {} From f6f93ef164f9a69739277ab06593df81b2fe841b Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 15:23:03 -0700 Subject: [PATCH 31/40] update checkin status --- plugins/inputs/checkin_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/checkin_status.go b/plugins/inputs/checkin_status.go index a63fe3f89d..bbcc35b0dc 100644 --- a/plugins/inputs/checkin_status.go +++ b/plugins/inputs/checkin_status.go @@ -49,5 +49,5 @@ func (s *CheckinStatusSensor) FormattedLatestBuffer() string { return "\n" + in.Input + "\n" } -func (s *CheckinStatusSensor) TriggersTick() bool { return false } +func (s *CheckinStatusSensor) TriggersTick() bool { return true } func (s *CheckinStatusSensor) Stop() {} From b3e6243f86a543feaf1e2ae4378725cd97704863 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 15:29:14 -0700 Subject: [PATCH 32/40] update checkin status --- plugins/inputs/checkin_status.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/checkin_status.go b/plugins/inputs/checkin_status.go index bbcc35b0dc..eed3f77e51 100644 --- a/plugins/inputs/checkin_status.go +++ b/plugins/inputs/checkin_status.go @@ -49,5 +49,8 @@ func (s *CheckinStatusSensor) FormattedLatestBuffer() string { return "\n" + in.Input + "\n" } -func (s *CheckinStatusSensor) TriggersTick() bool { return true } +func (s *CheckinStatusSensor) TriggersTick() bool { + in := providers.IO().GetInput("CheckinStatus") + return in != nil && in.Input != "" +} func (s *CheckinStatusSensor) Stop() {} From adc85664367717d0d3a3f0f3be4bd94839b639a4 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 16:04:21 -0700 Subject: [PATCH 33/40] fix new attendee greeting --- plugins/backgrounds/luma/guest_lingering.go | 1 + plugins/backgrounds/luma/luma_checkin.go | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/backgrounds/luma/guest_lingering.go b/plugins/backgrounds/luma/guest_lingering.go index 6885a07814..cf84d8efaf 100644 --- a/plugins/backgrounds/luma/guest_lingering.go +++ b/plugins/backgrounds/luma/guest_lingering.go @@ -76,6 +76,7 @@ func NewGuestLingering(configMap map[string]any) (bg.Background, error) { period: time.Duration(cfg.PollSec * float64(time.Second)), gracePeriod: time.Duration(cfg.GracePeriodSec * float64(time.Second)), minArea: cfg.MinFaceArea, + lastHandled: time.Now(), }, nil } diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index 11dea0be69..c33989fd28 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -75,6 +75,7 @@ func LumaCheckin(configMap map[string]any) (bg.Background, error) { period: time.Duration(cfg.PollSec * float64(time.Second)), gracePeriod: time.Duration(cfg.GracePeriodSec * float64(time.Second)), minArea: cfg.FaceMinArea, + lastHandled: time.Now(), }, nil } From ab1aafd6ffefab0e56c7833f54acd7630717cabc Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 16:29:51 -0700 Subject: [PATCH 34/40] add logs --- plugins/backgrounds/luma/guest_lingering.go | 18 +++++------ plugins/backgrounds/luma/luma_checkin.go | 34 +++++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/plugins/backgrounds/luma/guest_lingering.go b/plugins/backgrounds/luma/guest_lingering.go index cf84d8efaf..6a379d8dc6 100644 --- a/plugins/backgrounds/luma/guest_lingering.go +++ b/plugins/backgrounds/luma/guest_lingering.go @@ -107,21 +107,19 @@ func (g *GuestLingering) Run(ctx context.Context) { return } - // Check if the primary guest's face is still present. + // Check if the primary guest's face is still present (by track_id or any large face as fallback). var guestPresent bool for _, face := range snap.Faces { - if primaryTrackID > 0 && face.TrackID == primaryTrackID && float64(face.Area) >= g.minArea { + if float64(face.Area) < g.minArea { + continue + } + if primaryTrackID > 0 && face.TrackID == primaryTrackID { guestPresent = true break } - } - // Fallback: if no track_id match (e.g. ID reassigned), use largest face. - if !guestPresent && primaryTrackID == 0 { - for _, face := range snap.Faces { - if float64(face.Area) >= g.minArea { - guestPresent = true - break - } + if primaryTrackID == 0 { + guestPresent = true + break } } diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index c33989fd28..874d9a8ac3 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -80,28 +80,22 @@ func LumaCheckin(configMap map[string]any) (bg.Background, error) { } func (c *CheckinComplete) Run(ctx context.Context) { - // Step 1: pick up the latest successful check-in. Skip when there is none yet - // or when we've already acted on it. checkin := luma.LastCheckIn() if checkin == nil || !checkin.Time.After(c.lastHandled) { util.Sleep(ctx, c.period) return } - // Step 2: wait a grace period after the check-in before checking for face - // departure, so the greeting has time to play. if time.Since(checkin.Time) < c.gracePeriod { util.Sleep(ctx, c.period) return } - // Read the primary guest's track_id stored by FaceSizeWatch. var primaryTrackID int if in := providers.IO().GetInput("PrimaryGuestTrackID"); in != nil && in.Input != "" { primaryTrackID, _ = strconv.Atoi(in.Input) } - // Step 3: check if the primary guest's face is gone. snap, err := c.face.FetchSnapshot(ctx) if err != nil { if ctx.Err() == nil { @@ -111,23 +105,31 @@ func (c *CheckinComplete) Run(ctx context.Context) { return } + c.log.Debug("departure check", + zap.Int("primary_track_id", primaryTrackID), + zap.Int("num_faces", len(snap.Faces)), + zap.String("checkin_name", checkin.Name), + ) + + // Check if primary guest is still present by track_id or fallback to any large face. for _, face := range snap.Faces { - if primaryTrackID > 0 && face.TrackID == primaryTrackID && float64(face.Area) >= c.minArea { + if float64(face.Area) < c.minArea { + continue + } + if primaryTrackID > 0 && face.TrackID == primaryTrackID { util.Sleep(ctx, c.period) return } - } - // Fallback: if no track_id match, check if any large face remains. - if primaryTrackID == 0 { - for _, face := range snap.Faces { - if float64(face.Area) >= c.minArea { - util.Sleep(ctx, c.period) - return - } + if primaryTrackID == 0 { + util.Sleep(ctx, c.period) + return } } - c.log.Info("guest departed after successful check-in, triggering transition", zap.String("name", checkin.Name)) + c.log.Info("guest departed after successful check-in, triggering transition", + zap.String("name", checkin.Name), + zap.Int("primary_track_id", primaryTrackID), + ) providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) c.lastHandled = checkin.Time From ee59c1e875681783a3f7ba7a33f1566066544d3e Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 16:55:46 -0700 Subject: [PATCH 35/40] fix lingering face logic --- plugins/inputs/checkin_status.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/checkin_status.go b/plugins/inputs/checkin_status.go index eed3f77e51..022376e9da 100644 --- a/plugins/inputs/checkin_status.go +++ b/plugins/inputs/checkin_status.go @@ -2,6 +2,7 @@ package inputs import ( "context" + "time" "go.uber.org/zap" @@ -28,7 +29,22 @@ func (s *CheckinStatusSensor) Listen(ctx context.Context) (<-chan any, error) { out := make(chan any) go func() { defer close(out) - <-ctx.Done() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + in := providers.IO().GetInput("CheckinStatus") + if in != nil && in.Input != "" { + select { + case out <- in.Input: + default: + } + } + case <-ctx.Done(): + return + } + } }() return out, nil } @@ -37,8 +53,12 @@ func (s *CheckinStatusSensor) Poll(_ context.Context) (any, error) { return nil, nil } -func (s *CheckinStatusSensor) RawToText(_ context.Context, _ any) (*inputs.Message, error) { - return nil, nil +func (s *CheckinStatusSensor) RawToText(_ context.Context, raw any) (*inputs.Message, error) { + msg, ok := raw.(string) + if !ok || msg == "" { + return nil, nil + } + return &inputs.Message{Message: msg}, nil } func (s *CheckinStatusSensor) FormattedLatestBuffer() string { @@ -49,8 +69,5 @@ func (s *CheckinStatusSensor) FormattedLatestBuffer() string { return "\n" + in.Input + "\n" } -func (s *CheckinStatusSensor) TriggersTick() bool { - in := providers.IO().GetInput("CheckinStatus") - return in != nil && in.Input != "" -} +func (s *CheckinStatusSensor) TriggersTick() bool { return true } func (s *CheckinStatusSensor) Stop() {} From 475eface13390f3b442fc618e3265732799c483b Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 17:07:43 -0700 Subject: [PATCH 36/40] fix lingering face logic --- plugins/backgrounds/luma/guest_lingering.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/backgrounds/luma/guest_lingering.go b/plugins/backgrounds/luma/guest_lingering.go index 6a379d8dc6..ca309c14a7 100644 --- a/plugins/backgrounds/luma/guest_lingering.go +++ b/plugins/backgrounds/luma/guest_lingering.go @@ -65,6 +65,8 @@ func NewGuestLingering(configMap map[string]any) (bg.Background, error) { Timeout: 2 * time.Second, }) + providers.IO().AddInput("CheckinStatus", "", time.Now()) + log.Info("initialized", zap.Float64("min_face_area", cfg.MinFaceArea), zap.Float64("grace_period_sec", cfg.GracePeriodSec), From 1f273fbfcfbb8f33d144bdd548bb415fbbfa80f1 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 17:45:08 -0700 Subject: [PATCH 37/40] clear the queue after mode sqitch --- plugins/backgrounds/luma/luma_checkin.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index 874d9a8ac3..aeaf41fe38 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -12,6 +12,7 @@ import ( "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" "github.com/openmind/om1/internal/providers/luma" + "github.com/openmind/om1/internal/providers/tts" "github.com/openmind/om1/internal/util" ) @@ -130,6 +131,7 @@ func (c *CheckinComplete) Run(ctx context.Context) { zap.String("name", checkin.Name), zap.Int("primary_track_id", primaryTrackID), ) + tts.RequestInterrupt() providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) c.lastHandled = checkin.Time From d6b374e0ea7ab34d2348a7cb5aa4e08dec9494c1 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 17:51:50 -0700 Subject: [PATCH 38/40] fix luma hook --- internal/hooks/luma_hook.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/hooks/luma_hook.go b/internal/hooks/luma_hook.go index 1085d925e0..834c5c05b6 100644 --- a/internal/hooks/luma_hook.go +++ b/internal/hooks/luma_hook.go @@ -8,7 +8,7 @@ func init() { RegisterHook("luma_hook", "luma_intro_hook", (*Runner).lumaIntroHook) } -const defaultLumaHelp = "Please show me your Luma QR code to check in. If you didn't register on Luma, just tell me your name." +const defaultLumaHelp = "Are you registered on Luma?" const defaultLumaPrompt = "You are {robot_name}, a friendly robot welcoming a guest to an event. " + "The current time is {current_time}. " + @@ -17,7 +17,7 @@ const defaultLumaPrompt = "You are {robot_name}, a friendly robot welcoming a gu "Make it feel personal and present by naturally referencing something specific from what you see; never invent anything. " + "If a specific person is recognized ({closest_name}), greet them by name; otherwise greet generically. " + "{memory}" + - "Finish by asking the guest to show their Luma QR code so you can check them in, and tell them that if they did not register on Luma they can simply say their name instead, for example: \"{help_message}\". " + + "Finish by asking the guest whether they are registered on Luma, for example: \"{help_message}\". " + "Respond with only the greeting text, with no quotes or commentary." func (r *Runner) lumaIntroHook(ctx context.Context, cfg, vars map[string]any) error { From 6a5e2dc16eb5882431633b00301cd9751de41754 Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 18:05:39 -0700 Subject: [PATCH 39/40] fix lingering person speech generateion --- plugins/backgrounds/luma/guest_lingering.go | 7 +++++++ plugins/backgrounds/luma/luma_checkin.go | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/backgrounds/luma/guest_lingering.go b/plugins/backgrounds/luma/guest_lingering.go index ca309c14a7..2acacfcff4 100644 --- a/plugins/backgrounds/luma/guest_lingering.go +++ b/plugins/backgrounds/luma/guest_lingering.go @@ -12,6 +12,7 @@ import ( "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" "github.com/openmind/om1/internal/providers/luma" + "github.com/openmind/om1/internal/providers/tts" "github.com/openmind/om1/internal/util" ) @@ -126,6 +127,12 @@ func (g *GuestLingering) Run(ctx context.Context) { } if guestPresent { + // Only emit a fresh nudge when TTS is idle, so nudges never pile up in the + // queue while the robot is still speaking or has speech queued. + if tts.Busy() { + util.Sleep(ctx, g.period) + return + } g.log.Info("guest lingering after check-in", zap.String("name", checkin.Name), zap.Int("primary_track_id", primaryTrackID), diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index aeaf41fe38..874d9a8ac3 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -12,7 +12,6 @@ import ( "github.com/openmind/om1/internal/logger" "github.com/openmind/om1/internal/providers" "github.com/openmind/om1/internal/providers/luma" - "github.com/openmind/om1/internal/providers/tts" "github.com/openmind/om1/internal/util" ) @@ -131,7 +130,6 @@ func (c *CheckinComplete) Run(ctx context.Context) { zap.String("name", checkin.Name), zap.Int("primary_track_id", primaryTrackID), ) - tts.RequestInterrupt() providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) c.lastHandled = checkin.Time From 0f59fa74e9ba8b027e44ecb4bc083ff3d0d2082e Mon Sep 17 00:00:00 2001 From: Prachi Sethi Date: Mon, 15 Jun 2026 18:15:33 -0700 Subject: [PATCH 40/40] fix lingering person speech generateion --- plugins/backgrounds/luma/luma_checkin.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/backgrounds/luma/luma_checkin.go b/plugins/backgrounds/luma/luma_checkin.go index 874d9a8ac3..95f2dc9b31 100644 --- a/plugins/backgrounds/luma/luma_checkin.go +++ b/plugins/backgrounds/luma/luma_checkin.go @@ -130,6 +130,8 @@ func (c *CheckinComplete) Run(ctx context.Context) { zap.String("name", checkin.Name), zap.Int("primary_track_id", primaryTrackID), ) + // Clear any lingering signal so no further nudge ticks fire while we transition. + providers.IO().AddInput("CheckinStatus", "", time.Now()) providers.ModeContext().Publish(map[string]any{"checkin_complete": true}) c.lastHandled = checkin.Time