Retrieve /match candidates per comparable name part

docs/settings.md

@@ -35,6 +35,6 @@ Yente features various configuration options related to data refresh and re-inde
 | `YENTE_MATCH_PAGE` | `5` | How many results to return per `/match` query by default. |
 | `YENTE_MAX_MATCHES` | `500` | How many results to return per `/match` query at most. |
 | `YENTE_MATCH_CANDIDATES` | `10` | How many candidates to retrieve from the search as a multiplier of the `/match` limit. Note that increasing this parameter will also increase query cost, as each of these candidates scored after retrieval from the index.|
-| `YENTE_MATCH_FUZZY` | `true` | Whether to run expensive Levenshtein queries inside ElasticSearch. |
+| `YENTE_MATCH_FUZZY` | `true` | Whether `/match` candidate retrieval adds an edit-distance (Levenshtein) clause per query name part, so that typos and minor spelling variants are still retrieved. Exact, known-name symbol and space-less name matching are always on. |
 | `YENTE_DELTA_UPDATES` | `true` | When set to `false` Yente will download the entire dataset when refreshing the index. |
 | `YENTE_STREAM_LOAD`   | `true`   | If set to `false`, will download the full data before indexing it. This can improve the stability of the indexer, especially when the network connection is a bit sketchy, but requires some local disk cache space.   |

tests/test_mappings.py

@@ -52,17 +52,15 @@ async def test_mappings_copy_to(search_provider):
         )
         assert len(search_result["hits"]["hits"]) == 1, "Failed to match on names"
 
-        # name_parts and name_phonetic are a bit of a special case, we syntesize them in the indexer
+        # name_parts and name_joined are a bit of a special case, we syntesize them in the indexer
         search_result = await search_provider.search(
             temp_index, {"bool": {"must": [{"match": {"name_parts": "Vladimir"}}]}}
         )
         assert len(search_result["hits"]["hits"]) == 1, "Failed to match on name_parts"
         search_result = await search_provider.search(
-            temp_index, {"bool": {"must": [{"match": {"name_phonetic": "FLTMR"}}]}}
-        )
-        assert len(search_result["hits"]["hits"]) == 1, (
-            "Failed to match on name_phonetic"
+            temp_index, {"bool": {"must": [{"term": {"name_joined": "vladimirputin"}}]}}
         )
+        assert len(search_result["hits"]["hits"]) == 1
 
         # Try to match on the countries field, which is a type field that is populated by copy_to from citizenship
         search_result = await search_provider.search(
@@ -149,13 +147,15 @@ def test_name_symbols_indexed_org(search_provider):
     assert "DOMAIN:BANK" in doc["name_symbols"]
 
 
-def test_name_phonetic_indexed(search_provider):
+def test_name_joined_indexed():
     entity = Entity.from_dict(
         {
             "id": "Q7747",
             "schema": "Person",
             "properties": {
                 "name": ["Vladimir V. Putin"],
+                "alias": ["Владимир Путин"],
+                "weakAlias": ["Vova"],
                 "citizenship": ["ru"],
                 "topics": ["sanction"],
             },
@@ -164,5 +164,20 @@ def test_name_phonetic_indexed(search_provider):
 
     doc = build_indexable_entity_doc(entity)
 
-    # Ensure that the "V." doesn't end up in the phonetics, it's too short.
-    assert set(doc["name_phonetic"]) == {"FLTMR", "PTN"}
+    assert set(doc["name_joined"]) == {"vladimirvputin", "vladimirputin"}
+    assert "vova" in doc["name_parts"]
+    assert "name_phonetic" not in doc
+
+
+def test_name_joined_indexed_org():
+    entity = Entity.from_dict(
+        {
+            "id": "Q1234",
+            "schema": "Company",
+            "properties": {"name": ["Al-Qaeda Trading LLC"]},
+        }
+    )
+
+    doc = build_indexable_entity_doc(entity)
+
+    assert "alqaedatradingllc" in doc["name_joined"]

tests/test_match.py

@@ -216,37 +216,25 @@ def test_id_pass_through():
 
 @pytest.mark.usefixtures("zala_test_dataset")
 def test_match_name_without_spaces():
-    # A name with spaces omitted ("alexandervyacheslavovichzakharov") is a single token to
-    # the query analyzer, so the per-token names match can't bridge the gap to the
-    # separate indexed tokens. We solve this with the character n-gram sub-field on the
-    # names field, which lets the space-less token still match the indexed name via shared
-    # n-grams, and this test verifies that.
-    #
-    # The bogus "foo bar" name is just there to keep the query from being treated as a
-    # single-word ("short") query, for which we always fall back to n-gram matching
-    # regardless of the fuzzy setting. That way we can show the n-gram path is what makes
-    # the space-less match work, by checking it disappears when fuzzy matching is off.
+    # A name with its spaces omitted is a single token to the name analysis, so no
+    # per-part clause can reach the indexed name. The indexed space-less form of each
+    # name (name_joined) bridges that exactly, independent of the fuzzy setting.
     query = {
         "queries": {
             "a": {
                 "schema": "Person",
-                "properties": {"name": ["alexandervyacheslavovichzakharov", "foo bar"]},
+                "properties": {"name": ["alexandervyacheslavovichzakharov"]},
             }
         }
     }
 
-    with mock.patch("yente.settings.MATCH_FUZZY", False):
-        resp = client.post("/match/zala", json=query)
-        assert resp.status_code == 200, resp.text
-        res = resp.json()["responses"]["a"]
-        assert len(res["results"]) == 0
-
-    with mock.patch("yente.settings.MATCH_FUZZY", True):
-        resp = client.post("/match/zala", json=query)
-        assert resp.status_code == 200, resp.text
-        res = resp.json()["responses"]["a"]
-        assert len(res["results"]) > 0
-        assert res["results"][0]["id"] == "NK-aU5ybkbRFJucf8YMwsJvDw"
+    for fuzzy in (False, True):
+        with mock.patch("yente.settings.MATCH_FUZZY", fuzzy):
+            resp = client.post("/match/zala", json=query)
+            assert resp.status_code == 200, resp.text
+            res = resp.json()["responses"]["a"]
+            assert len(res["results"]) > 0
+            assert res["results"][0]["id"] == "NK-aU5ybkbRFJucf8YMwsJvDw"
 
 
 @pytest.mark.usefixtures("zala_test_dataset")

tests/test_queries.py

@@ -0,0 +1,191 @@
+import itertools
+import string
+from typing import Any
+from unittest import mock
+
+from yente.data.entity import Entity
+from yente.search.queries import (
+    FUZZY_BOOST,
+    JOINED_BOOST,
+    MAX_PARTS,
+    MAX_SYMBOLS_PER_PART,
+    SYMBOL_BOOST,
+    WEAK_ALIAS_BOOST,
+    names_query,
+)
+
+MANY_NAMES = [
+    "Alexander Vyacheslavovich ZAKHAROV",
+    "Aleksandr Vyacheslavovich Zakharov",
+    "Александр Вячеславович Захаров",
+    "Александр ЗАХАРОВ",
+    "Захаров Александр Вячеславович",
+    "Zakharov Aleksandr Vyacheslavovich",
+    "Aleksandr Vjačeslavovič Zacharov",
+]
+
+
+def make_entity(id_: str, schema: str, properties: dict[str, Any]) -> Entity:
+    # The name analysis is cached on the entity ID, so every test entity needs
+    # its own ID.
+    return Entity.from_dict(
+        {"id": id_, "schema": schema, "properties": properties, "datasets": ["test"]}
+    )
+
+
+def part_clauses(shoulds: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
+    """Map each queried name part to the channel clauses inside its dis_max."""
+    parts: dict[str, list[dict[str, Any]]] = {}
+    for clause in shoulds:
+        if "dis_max" not in clause:
+            continue
+        channels = clause["dis_max"]["queries"]
+        parts[channels[0]["term"]["name_parts"]["value"]] = channels
+    return parts
+
+
+def channel(channels: list[dict[str, Any]], kind: str) -> dict[str, Any] | None:
+    for clause in channels:
+        if kind in clause:
+            return clause
+    return None
+
+
+def test_one_clause_per_unique_part():
+    entity = make_entity("q-many", "Person", {"name": MANY_NAMES})
+    shoulds = names_query(entity)
+    parts = part_clauses(shoulds)
+    assert set(parts) == {
+        "alexander",
+        "aleksandr",
+        "vyacheslavovich",
+        "vjaceslavovic",
+        "vaceslavovic",
+        "zakharov",
+        "zaharov",
+        "zacharov",
+    }
+    for channels in parts.values():
+        assert channels[0]["term"]["name_parts"]["boost"] == 1.0
+    dis_maxes = [c for c in shoulds if "dis_max" in c]
+    assert len(dis_maxes) == len(parts)
+    for clause in dis_maxes:
+        assert clause["dis_max"]["tie_breaker"] == 0.0
+
+
+def test_fuzzy_channel():
+    entity = make_entity("q-putin", "Person", {"name": ["Vladimir Putin"]})
+    with mock.patch("yente.settings.MATCH_FUZZY", True):
+        parts = part_clauses(names_query(entity))
+    fuzzy = channel(parts["putin"], "constant_score")
+    assert fuzzy == {
+        "constant_score": {
+            "filter": {
+                "fuzzy": {
+                    "name_parts": {
+                        "value": "putin",
+                        "fuzziness": "AUTO",
+                        "prefix_length": 1,
+                        "max_expansions": 200,
+                    }
+                }
+            },
+            "boost": FUZZY_BOOST,
+        }
+    }
+
+
+def test_fuzzy_channel_off():
+    entity = make_entity("q-putin-nofuzzy", "Person", {"name": ["Vladimir Putin"]})
+    with mock.patch("yente.settings.MATCH_FUZZY", False):
+        parts = part_clauses(names_query(entity))
+    assert set(parts) == {"vladimir", "putin"}
+    for channels in parts.values():
+        assert channel(channels, "constant_score") is None
+        assert channel(channels, "term") is not None
+
+
+def test_fuzzy_channel_skips_short_parts():
+    entity = make_entity("q-li", "Person", {"name": ["Li Na"]})
+    with mock.patch("yente.settings.MATCH_FUZZY", True):
+        parts = part_clauses(names_query(entity))
+    assert set(parts) == {"li", "na"}
+    for channels in parts.values():
+        assert channel(channels, "constant_score") is None
+
+
+def test_symbol_channel():
+    entity = make_entity("q-putin-sym", "Person", {"name": ["Vladimir Putin"]})
+    parts = part_clauses(names_query(entity))
+    symbols = channel(parts["putin"], "dis_max")
+    assert symbols is not None
+    assert symbols["dis_max"]["boost"] == SYMBOL_BOOST
+    terms = symbols["dis_max"]["queries"]
+    assert {"term": {"name_symbols": {"value": "NAME:KHUYLO", "boost": 1.0}}} in terms
+
+
+def test_symbol_channel_absent_for_untagged_part():
+    entity = make_entity("q-xyzzy", "Person", {"name": ["Xyzzyq Plughz"]})
+    parts = part_clauses(names_query(entity))
+    for channels in parts.values():
+        assert channel(channels, "dis_max") is None
+
+
+def test_symbol_channel_cap():
+    entity = make_entity("q-li-cap", "Person", {"name": ["Li Na"]})
+    parts = part_clauses(names_query(entity))
+    li_symbols = channel(parts["li"], "dis_max")
+    assert li_symbols is not None
+    assert len(li_symbols["dis_max"]["queries"]) == MAX_SYMBOLS_PER_PART
+
+    with mock.patch("yente.search.queries.MAX_SYMBOLS_PER_PART", 2):
+        parts = part_clauses(names_query(entity))
+    li_symbols = channel(parts["li"], "dis_max")
+    assert li_symbols is not None
+    assert len(li_symbols["dis_max"]["queries"]) == 2
+
+
+def test_joined_clause():
+    entity = make_entity(
+        "q-joined", "Person", {"name": ["Vladimir Putin"], "alias": ["Vova Putin"]}
+    )
+    shoulds = names_query(entity)
+    joined = [c for c in shoulds if "terms" in c]
+    assert joined == [
+        {
+            "terms": {
+                "name_joined": ["vladimirputin", "vovaputin"],
+                "boost": JOINED_BOOST,
+            }
+        }
+    ]
+
+
+def test_weak_alias_clause():
+    entity = make_entity(
+        "q-weak", "Person", {"name": ["Vladimir Putin"], "weakAlias": ["Vova"]}
+    )
+    shoulds = names_query(entity)
+    assert {"term": {"name_parts": {"value": "vova", "boost": WEAK_ALIAS_BOOST}}} in (
+        shoulds
+    )
+    assert "vova" not in part_clauses(shoulds)
+
+
+def test_parts_cap():
+    tokens = ["".join(t) for t in itertools.product(string.ascii_lowercase, repeat=3)]
+    entity = make_entity("q-big", "Person", {"name": [" ".join(tokens[:150])]})
+    shoulds = names_query(entity)
+    parts = part_clauses(shoulds)
+    assert len(parts) == MAX_PARTS
+    assert len([c for c in shoulds if "dis_max" in c]) == MAX_PARTS
+
+
+def test_primary_name_parts_come_first():
+    entity = make_entity(
+        "q-order",
+        "Person",
+        {"name": ["Zebediah Quill"], "alias": ["Aaron Abbott"]},
+    )
+    parts = list(part_clauses(names_query(entity)))
+    assert parts[:2] == ["zebediah", "quill"]

yente/search/indexer.py

@@ -29,8 +29,8 @@
 )
 from yente.search.mapping import (
     INDEX_SETTINGS,
+    NAME_JOINED_FIELD,
     NAME_PART_FIELD,
-    NAME_PHONETIC_FIELD,
     NAME_SYMBOLS_FIELD,
     make_entity_mapping,
 )
@@ -110,22 +110,24 @@ def build_indexable_entity_doc(entity: Entity) -> dict[str, Any]:
     doc["entity_values_count"] = sum([len(v) for v in doc["properties"].values()])
 
     name_parts: set[str] = set()
-    name_phonemes: set[str] = set()
     name_symbols: set[str] = set()
-    for name in entity_names(entity, infer_initials=False, consolidate=False):
+    name_joined: set[str] = set()
+    names = entity_names(
+        entity, infer_initials=False, phonetics=False, consolidate=False
+    )
+    for name in names:
         name_symbols.update(index_symbols(name.symbols))
-        for part in name.parts:
-            name_parts.add(part.comparable)
-            phoneme = part.metaphone
-            if phoneme is not None and len(phoneme) > 2:
-                name_phonemes.add(phoneme)
+        comparables = [part.comparable for part in name.parts]
+        name_parts.update(comparables)
+        if len(comparables) > 0:
+            name_joined.add("".join(comparables))
 
     for weak in entity_weak_names(entity):
         name_parts.add(weak)
 
     doc[NAME_PART_FIELD] = list(name_parts)
-    doc[NAME_PHONETIC_FIELD] = list(name_phonemes)
     doc[NAME_SYMBOLS_FIELD] = list(name_symbols)
+    doc[NAME_JOINED_FIELD] = list(name_joined)
     if registry.date.group is not None:
         doc[registry.date.group] = expand_dates(doc.pop(registry.date.group, []))
     doc["text"] = entity.pop("indexText")