add speak detection in om1

config/greeting_conversation.json5

@@ -116,6 +116,36 @@ Speaking Action Selection:\n\
 - If a response primarily requires a special gesture (such as SHAKE_HAND, FACE_WAVE, SALUTE, HEART, SHRUG, HANDS_UP, COME_CLOSER, or PUSH), use that gesture instead of a talking action.\n\
 - Otherwise, prefer the appropriate talking_xs action while speaking rather than STAND_STILL.\n\
 \n\
+Knowing Who You Are Talking To:\n\
+- Every round you are given a FacePresence line and a Speaker line.\n\
+- The Speaker line is measured by an audio-visual model, not guessed. Trust it over face position.\n\
+- The face tagged [SPEAKING NOW] is the person who just spoke. Faces tagged [not speaking] did not.\n\
+- Attribute everything you are told - names, preferences, corrections - to the Speaker, never to the nearest face.\n\
+- When the Speaker line says unknown AND several faces are listed, you do NOT know who spoke. Do not attach anything to a particular person. With a single face there is no ambiguity to resolve.\n\
+- Not having a name is not the same as not having an identity. An enrolled but unnamed person is still a person the robot remembers; talk to them and let the conversation be recorded as normal.\n\
+- Never pressure anyone for their name. If they decline or change the subject, drop it and carry on - the conversation is still remembered either way.\n\
+- If a face is marked GUESS, there is no speech detection running. Treat identity as unconfirmed.\n\
+\n\
+Remembering Names:\n\
+- When the person speaking tells you their name (for example 'my name is X', 'I am X', 'call me X'), record it.\n\
+- Do this by emitting the face_memory action with op set to set_name and to_id set to the name they gave.\n\
+- ALWAYS op set_name, never op selfie. Everyone the robot can see already has an identity, even the ones shown as newcomer or anon_xxxx; they need a name attached, not a new enrolment.\n\
+- Write to_id as lowercase Latin letters with dashes for spaces: Wendy becomes wendy, Li Fan becomes li-fan, Jerin Peter becomes jerin-peter.\n\
+- For a name with no Latin spelling, ask once how they would spell it in English or Pinyin, and use that.\n\
+- Only ever record a name for the person the Speaker line identifies.\n\
+- If the FacePresence line shows only ONE face, an unknown Speaker changes nothing: there is nobody else it could have been. Record the name.\n\
+- Only when SEVERAL faces are listed does an unknown Speaker matter. Then ask them once to look at you and say it again, and keep talking with them regardless.\n\
+- Confirm naturally in your reply, for example 'Nice to meet you, X'. Never read the action out loud.\n\
+- If they are correcting a name you already have, use op correct_identity with from_id and to_id instead.\n\
+- Never invent a name, and never re-record a name that is already correct.\n\
+\n\
+When You Cannot Save Their Name:\n\
+- The Speaker line will sometimes say their face cannot be enrolled from where they are. Saving a name is impossible until that changes.\n\
+- You may invite them ONCE, warmly and in passing, to step a little closer or turn towards you - for example 'I can hear you, but I can't quite see you - would you mind stepping a little closer?'\n\
+- Ask at most once per person. If they do not move, do not repeat it, do not explain the camera, and do not keep bringing it up.\n\
+- Then simply carry on the conversation as normal. Not being able to save someone's name is not a problem you need to solve, and it is not their problem to fix.\n\
+- Never make it a condition of talking to them, never imply they are doing something wrong, and never ask them to move for any other reason.\n\
+\n\
 You should prioritize safe, comfortable, and positive human interaction.",
       hertz: 0.001,
       agent_inputs: [
@@ -136,6 +166,9 @@ You should prioritize safe, comfortable, and positive human interaction.",
         },
         {
           type: "FacePresence",
+          config: {
+            speaker_detection: true,
+          },
         },
         {
           type: "GreetingStatus",
@@ -265,6 +298,36 @@ Speaking Action Selection:\n\
 - If a response primarily requires a special gesture (such as SHAKE_HAND, FACE_WAVE, SALUTE, HEART, SHRUG, HANDS_UP, COME_CLOSER, or PUSH), use that gesture instead of a talking action.\n\
 - Otherwise, prefer the appropriate talking_xs action while speaking rather than STAND_STILL.\n\
 \n\
+Knowing Who You Are Talking To:\n\
+- Every round you are given a FacePresence line and a Speaker line.\n\
+- The Speaker line is measured by an audio-visual model, not guessed. Trust it over face position.\n\
+- The face tagged [SPEAKING NOW] is the person who just spoke. Faces tagged [not speaking] did not.\n\
+- Attribute everything you are told - names, preferences, corrections - to the Speaker, never to the nearest face.\n\
+- When the Speaker line says unknown AND several faces are listed, you do NOT know who spoke. Do not attach anything to a particular person. With a single face there is no ambiguity to resolve.\n\
+- Not having a name is not the same as not having an identity. An enrolled but unnamed person is still a person the robot remembers; talk to them and let the conversation be recorded as normal.\n\
+- Never pressure anyone for their name. If they decline or change the subject, drop it and carry on - the conversation is still remembered either way.\n\
+- If a face is marked GUESS, there is no speech detection running. Treat identity as unconfirmed.\n\
+\n\
+Remembering Names:\n\
+- When the person speaking tells you their name (for example 'my name is X', 'I am X', 'call me X'), record it.\n\
+- Do this by emitting the face_memory action with op set to set_name and to_id set to the name they gave.\n\
+- ALWAYS op set_name, never op selfie. Everyone the robot can see already has an identity, even the ones shown as newcomer or anon_xxxx; they need a name attached, not a new enrolment.\n\
+- Write to_id as lowercase Latin letters with dashes for spaces: Wendy becomes wendy, Li Fan becomes li-fan, Jerin Peter becomes jerin-peter.\n\
+- For a name with no Latin spelling, ask once how they would spell it in English or Pinyin, and use that.\n\
+- Only ever record a name for the person the Speaker line identifies.\n\
+- If the FacePresence line shows only ONE face, an unknown Speaker changes nothing: there is nobody else it could have been. Record the name.\n\
+- Only when SEVERAL faces are listed does an unknown Speaker matter. Then ask them once to look at you and say it again, and keep talking with them regardless.\n\
+- Confirm naturally in your reply, for example 'Nice to meet you, X'. Never read the action out loud.\n\
+- If they are correcting a name you already have, use op correct_identity with from_id and to_id instead.\n\
+- Never invent a name, and never re-record a name that is already correct.\n\
+\n\
+When You Cannot Save Their Name:\n\
+- The Speaker line will sometimes say their face cannot be enrolled from where they are. Saving a name is impossible until that changes.\n\
+- You may invite them ONCE, warmly and in passing, to step a little closer or turn towards you - for example 'I can hear you, but I can't quite see you - would you mind stepping a little closer?'\n\
+- Ask at most once per person. If they do not move, do not repeat it, do not explain the camera, and do not keep bringing it up.\n\
+- Then simply carry on the conversation as normal. Not being able to save someone's name is not a problem you need to solve, and it is not their problem to fix.\n\
+- Never make it a condition of talking to them, never imply they are doing something wrong, and never ask them to move for any other reason.\n\
+\n\
 You should prioritize safe, comfortable, and positive human interaction.",
       hertz: 0.001,
       agent_inputs: [
@@ -279,6 +342,9 @@ You should prioritize safe, comfortable, and positive human interaction.",
         },
         {
           type: "FacePresence",
+          config: {
+            speaker_detection: true,
+          },
         },
         {
           type: "VLMBackground",

internal/actions/schema.go

@@ -60,6 +60,11 @@ func BuildSchema(llmLabel, description string, inputExample any) map[string]any
 		field := inputType.Field(i)
 
 		propertyName := field.Tag.Get("json")
+		optional := false
+		if idx := strings.Index(propertyName, ","); idx >= 0 {
+			optional = strings.Contains(propertyName[idx:], "omitempty")
+			propertyName = propertyName[:idx]
+		}
 		if propertyName == "" {
 			propertyName = strings.ToLower(field.Name)
 		}
@@ -71,7 +76,9 @@ func BuildSchema(llmLabel, description string, inputExample any) map[string]any
 		}
 
 		properties[propertyName] = BuildPropertySchema(field.Type, fieldDescription)
-		required = append(required, propertyName)
+		if !optional {
+			required = append(required, propertyName)
+		}
 	}
 
 	return map[string]any{

internal/providers/face_presence.go

@@ -81,6 +81,7 @@ type FaceEntry struct {
 	CreatedAgoSec  *float64 `json:"created_ago_sec"`   // UUID age in seconds; null for unknown
 	LastSeenAgoSec *float64 `json:"last_seen_ago_sec"` // sticky session-start gap; null until first confident match
 	LastSeenISO    *string  `json:"last_seen_iso"`     // ISO timestamp of previous sighting
+	Enrolling      bool     `json:"enrolling"`
 }
 
 // PresenceSnapshot is the parsed /who response.
@@ -174,6 +175,10 @@ func (p *FacePresenceProvider) FetchSnapshot(ctx context.Context) (PresenceSnaps
 // ToText renders the snapshot into one LLM-readable line.
 // Returns "" when no actionable faces are present.
 func (s *PresenceSnapshot) ToText() string {
+	return s.toTextWithSpeaker(Speaker().Latest(), Speaker().Available())
+}
+
+func (s *PresenceSnapshot) toTextWithSpeaker(spk *SpeakerResult, available bool) string {
 	if s == nil || len(s.Faces) == 0 {
 		return ""
 	}
@@ -188,11 +193,18 @@ func (s *PresenceSnapshot) ToText() string {
 		return faces[i].TrackID < faces[j].TrackID
 	})
 
+	speakingTrack := -1
+	if spk.Identified() {
+		speakingTrack = spk.TrackID
+	}
+
 	var kept []*FaceEntry
 	for i := range faces {
 		f := &faces[i]
 		if f.Name == "unknown" || f.Name == "" {
-			continue
+			if f.TrackID != speakingTrack || speakingTrack < 0 {
+				continue
+			}
 		}
 		kept = append(kept, f)
 	}
@@ -211,20 +223,115 @@ func (s *PresenceSnapshot) ToText() string {
 	descriptor += " (nearest first; nearest face is closest to the camera and most likely addressing the robot)"
 
 	var parts []string
+	var speakerEntry string
+	speakerEnrolling := false
+	speakerUUID := ""
 	for _, f := range kept {
 		var entry string
-		if strings.HasPrefix(f.Name, "anon_") {
+		switch {
+		case f.Name == "unknown" || f.Name == "":
+			entry = "an unrecognised person"
+		case strings.HasPrefix(f.Name, "anon_"):
 			entry = formatAnonEntry(*f)
-		} else {
+		default:
 			entry = formatNamedEntry(*f)
 		}
-		if f == closest {
-			entry += " [closest, likely speaking]"
+		switch {
+		case speakingTrack >= 0 && f.TrackID == speakingTrack:
+			entry += " [SPEAKING NOW]"
+			speakerEntry = entry
+			speakerEnrolling = f.Enrolling
+			speakerUUID = f.UUID
+		case speakingTrack >= 0:
+			entry += " [not speaking]"
+		case f == closest:
+			entry += " [closest, likely speaking — GUESS, no speech detection]"
 		}
 		parts = append(parts, entry)
 	}
 
-	return fmt.Sprintf("%s — %s", descriptor, strings.Join(parts, ", "))
+	line := fmt.Sprintf("%s — %s", descriptor, strings.Join(parts, ", "))
+	if speakerUUID == "" {
+		speakerUUID = spk.identityUUID()
+	} else {
+		Speaker().NoteIdentity(speakingTrack, speakerUUID)
+	}
+	return line + speakerSuffix(
+		spk, available, speakingTrack, speakerEntry, speakerEnrolling, speakerUUID)
+}
+
+func speakerSuffix(
+	spk *SpeakerResult, available bool, speakingTrack int,
+	speakerEntry string, speakerEnrolling bool, speakerUUID string,
+) string {
+	if !available {
+		return "\nSpeaker: unknown (no active-speaker detection running; " +
+			"do NOT assume the nearest face is the one talking)"
+	}
+	if spk == nil {
+		if Speaker().Pending() {
+			return "\nSpeaker: still being resolved for this utterance — " +
+				"do not attribute it to anyone yet"
+		}
+		return "\nSpeaker: unknown (no utterance resolved yet)"
+	}
+	if speakingTrack < 0 {
+		return "\nSpeaker: unknown (nobody scored as speaking over the last utterance)"
+	}
+	if speakerUUID == "" && !speakerEnrolling {
+		return fmt.Sprintf(
+			"\nSpeaker: %s (track %d, confidence %.2f) — their face cannot be "+
+				"enrolled from where they are, so you cannot save a name for "+
+				"them yet. You may invite them once to come closer or face you; "+
+				"if they do not, let it go and keep talking normally.",
+			speakerName(spk), spk.TrackID, spk.Score)
+	}
+	if speakerEntry == "" {
+		return fmt.Sprintf(
+			"\nSpeaker: %s (track %d, confidence %.2f) — no longer visible",
+			speakerName(spk), spk.TrackID, spk.Score)
+	}
+	return fmt.Sprintf(
+		"\nSpeaker: %s (track %d, confidence %.2f) — attribute what was just said to THIS person",
+		speakerName(spk), spk.TrackID, spk.Score)
+}
+
+func (s *PresenceSnapshot) AttributedUser() (uuid string, name string, measured bool) {
+	if s == nil {
+		return "", "", false
+	}
+	if spk := Speaker().Latest(); spk.Identified() {
+		for i := range s.Faces {
+			f := &s.Faces[i]
+			if f.TrackID != spk.TrackID || f.UUID == "" {
+				continue
+			}
+			n := f.Name
+			if strings.HasPrefix(n, "anon_") || n == "unknown" {
+				n = ""
+			}
+			return f.UUID, n, true
+		}
+		if spk.UUID != "" {
+			n := spk.Name
+			if strings.HasPrefix(n, "anon_") || n == "unknown" {
+				n = ""
+			}
+			return spk.UUID, n, true
+		}
+	}
+	return s.ClosestUUID, s.ClosestName, false
+}
+
+func speakerName(spk *SpeakerResult) string {
+	switch {
+	case spk.Name != "" && !strings.HasPrefix(spk.Name, "anon_") && spk.Name != "unknown":
+		return spk.Name
+	case spk.UUID != "":
+		return "an enrolled but unnamed person"
+	default:
+		return "an unrecognised person"
+	}
 }
 
 // formatNamedEntry — three-tier label based on last-seen gap.