Skip to content

searchers

pgvector-backed SNOMED searchers for attribute and reference retrieval.

AbstractSnomedSearcher

Base class for pgvector-backed SNOMED searchers.

Opens the database connection and provides cost tracking and lifecycle management (close / context manager), matching the patterns used by PgvectorConceptSearcher and LlmMapper.

Source code in src/ariadne/hierarchy/searchers.py
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
class AbstractSnomedSearcher:
    """Base class for pgvector-backed SNOMED searchers.

    Opens the database connection and provides cost tracking and lifecycle
    management (``close`` / context manager), matching the patterns used by
    ``PgvectorConceptSearcher`` and ``LlmMapper``.
    """

    def __init__(self, hierarchy_settings: HierarchySettings):
        self.hierarchy_settings = hierarchy_settings
        conn_str = get_environment_variable("VOCAB_CONNECTION_STRING")
        conn_str = conn_str.replace("+psycopg", "").replace("+psycopg2", "")
        self.connection = psycopg.connect(conn_str, autocommit=True)
        try:
            register_vector(self.connection)
            with self.connection.cursor() as cur:
                cur.execute(f"SET hnsw.ef_search = {self.hierarchy_settings.retrieval.hnsw_ef_search}")
            self.schema = get_environment_variable("VOCAB_SCHEMA")
        except Exception:
            self.connection.close()
            raise
        self._cost = 0.0

    # -- lifecycle ----------------------------------------------------------

    def get_total_cost(self) -> float:
        """Return accumulated embedding cost (USD)."""
        return self._cost

    def close(self):
        """Close the database connection."""
        self.connection.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        self.close()

close()

Close the database connection.

Source code in src/ariadne/hierarchy/searchers.py
61
62
63
def close(self):
    """Close the database connection."""
    self.connection.close()

get_total_cost()

Return accumulated embedding cost (USD).

Source code in src/ariadne/hierarchy/searchers.py
57
58
59
def get_total_cost(self) -> float:
    """Return accumulated embedding cost (USD)."""
    return self._cost

SnomedAttributeSearcher

Bases: AbstractSnomedSearcher

Source code in src/ariadne/hierarchy/searchers.py
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
class SnomedAttributeSearcher(AbstractSnomedSearcher):

    def search(self, text: str, category_name: str, top_k: int | None = None) -> SearchResult:
        """Embed *text* and return the closest concepts in the given attribute category.

        Args:
            text: Free-text value to embed and search for.
            category_name: SNOMED attribute category filter.
            top_k: Maximum number of results (defaults to ``hierarchy_settings.retrieval.top_k_per_category``).

        Returns:
            SearchResult(dataframe, cost).
        """
        top_k = top_k if top_k is not None else self.hierarchy_settings.retrieval.top_k_per_category
        result = get_embedding_vectors([text])
        vector = result["embeddings"][0]
        cost = result["usage"]["total_cost_usd"]
        self._cost += cost

        query = sql.SQL("""
            WITH q AS (SELECT %s::vector AS vec)
            SELECT concept_id, concept_code, concept_name, attribute_category,
                   1 - (embedding <=> q.vec) AS similarity
            FROM {schema}.{table}, q
            WHERE attribute_category = %s
            ORDER BY embedding <=> q.vec
            LIMIT %s
        """).format(schema=sql.Identifier(self.schema), table=sql.Identifier("snomed_attribute"))
        with self.connection.cursor() as cur:
            cur.execute(query, (vector.tolist(), category_name, top_k))
            rows = cur.fetchall()

        if not rows:
            return SearchResult(pd.DataFrame(columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]), cost)

        return SearchResult(pd.DataFrame(rows, columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]), cost)

    def search_batch(
        self,
        texts_and_categories: list[tuple[str, str, str]],
        top_k: int | None = None,
    ) -> SearchBatchResult:
        """Embed all *texts* in a single API call and run per-category queries.

        Args:
            texts_and_categories: List of ``(attr_key, text, category_name)`` tuples.
            top_k: Number of candidates per category (defaults to ``hierarchy_settings.retrieval.top_k_per_category``).

        Returns:
            SearchBatchResult(results, total_cost).
        """
        top_k = top_k if top_k is not None else self.hierarchy_settings.retrieval.top_k_per_category
        if not texts_and_categories:
            return SearchBatchResult({}, 0.0)

        texts = [t[1] for t in texts_and_categories]
        result = get_embedding_vectors(texts)
        vectors = result["embeddings"]
        cost = result["usage"]["total_cost_usd"]
        self._cost += cost

        results: dict[str, pd.DataFrame] = {}
        for (attr_key, _text, category_name), vector in zip(texts_and_categories, vectors):
            query = sql.SQL("""
                WITH q AS (SELECT %s::vector AS vec)
                SELECT concept_id, concept_code, concept_name, attribute_category,
                       1 - (embedding <=> q.vec) AS similarity
                FROM {schema}.{table}, q
                WHERE attribute_category = %s
                ORDER BY embedding <=> q.vec
                LIMIT %s
            """).format(schema=sql.Identifier(self.schema), table=sql.Identifier("snomed_attribute"))
            with self.connection.cursor() as cur:
                cur.execute(query, (vector.tolist(), category_name, top_k))
                rows = cur.fetchall()

            if rows:
                results[attr_key] = pd.DataFrame(
                    rows, columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]
                )
            else:
                results[attr_key] = pd.DataFrame(
                    columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]
                )
        return SearchBatchResult(results, cost)

search(text, category_name, top_k=None)

Embed text and return the closest concepts in the given attribute category.

Parameters:

Name Type Description Default
text str

Free-text value to embed and search for.

required
category_name str

SNOMED attribute category filter.

required
top_k int | None

Maximum number of results (defaults to hierarchy_settings.retrieval.top_k_per_category).

None

Returns:

Type Description
SearchResult

SearchResult(dataframe, cost).

Source code in src/ariadne/hierarchy/searchers.py
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def search(self, text: str, category_name: str, top_k: int | None = None) -> SearchResult:
    """Embed *text* and return the closest concepts in the given attribute category.

    Args:
        text: Free-text value to embed and search for.
        category_name: SNOMED attribute category filter.
        top_k: Maximum number of results (defaults to ``hierarchy_settings.retrieval.top_k_per_category``).

    Returns:
        SearchResult(dataframe, cost).
    """
    top_k = top_k if top_k is not None else self.hierarchy_settings.retrieval.top_k_per_category
    result = get_embedding_vectors([text])
    vector = result["embeddings"][0]
    cost = result["usage"]["total_cost_usd"]
    self._cost += cost

    query = sql.SQL("""
        WITH q AS (SELECT %s::vector AS vec)
        SELECT concept_id, concept_code, concept_name, attribute_category,
               1 - (embedding <=> q.vec) AS similarity
        FROM {schema}.{table}, q
        WHERE attribute_category = %s
        ORDER BY embedding <=> q.vec
        LIMIT %s
    """).format(schema=sql.Identifier(self.schema), table=sql.Identifier("snomed_attribute"))
    with self.connection.cursor() as cur:
        cur.execute(query, (vector.tolist(), category_name, top_k))
        rows = cur.fetchall()

    if not rows:
        return SearchResult(pd.DataFrame(columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]), cost)

    return SearchResult(pd.DataFrame(rows, columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]), cost)

search_batch(texts_and_categories, top_k=None)

Embed all texts in a single API call and run per-category queries.

Parameters:

Name Type Description Default
texts_and_categories list[tuple[str, str, str]]

List of (attr_key, text, category_name) tuples.

required
top_k int | None

Number of candidates per category (defaults to hierarchy_settings.retrieval.top_k_per_category).

None

Returns:

Type Description
SearchBatchResult

SearchBatchResult(results, total_cost).

Source code in src/ariadne/hierarchy/searchers.py
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def search_batch(
    self,
    texts_and_categories: list[tuple[str, str, str]],
    top_k: int | None = None,
) -> SearchBatchResult:
    """Embed all *texts* in a single API call and run per-category queries.

    Args:
        texts_and_categories: List of ``(attr_key, text, category_name)`` tuples.
        top_k: Number of candidates per category (defaults to ``hierarchy_settings.retrieval.top_k_per_category``).

    Returns:
        SearchBatchResult(results, total_cost).
    """
    top_k = top_k if top_k is not None else self.hierarchy_settings.retrieval.top_k_per_category
    if not texts_and_categories:
        return SearchBatchResult({}, 0.0)

    texts = [t[1] for t in texts_and_categories]
    result = get_embedding_vectors(texts)
    vectors = result["embeddings"]
    cost = result["usage"]["total_cost_usd"]
    self._cost += cost

    results: dict[str, pd.DataFrame] = {}
    for (attr_key, _text, category_name), vector in zip(texts_and_categories, vectors):
        query = sql.SQL("""
            WITH q AS (SELECT %s::vector AS vec)
            SELECT concept_id, concept_code, concept_name, attribute_category,
                   1 - (embedding <=> q.vec) AS similarity
            FROM {schema}.{table}, q
            WHERE attribute_category = %s
            ORDER BY embedding <=> q.vec
            LIMIT %s
        """).format(schema=sql.Identifier(self.schema), table=sql.Identifier("snomed_attribute"))
        with self.connection.cursor() as cur:
            cur.execute(query, (vector.tolist(), category_name, top_k))
            rows = cur.fetchall()

        if rows:
            results[attr_key] = pd.DataFrame(
                rows, columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]
            )
        else:
            results[attr_key] = pd.DataFrame(
                columns=["concept_id", "concept_code", "concept_name", "attribute_category", "similarity"]
            )
    return SearchBatchResult(results, cost)

SnomedReferenceConceptVectorSearcher

Bases: AbstractSnomedSearcher

Reference searcher that builds few-shot reference examples directly from the OHDSI Vocabulary tables plus a configured concept-vector table restricted to S terms in SNOMED.

Source code in src/ariadne/hierarchy/searchers.py
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
class SnomedReferenceConceptVectorSearcher(AbstractSnomedSearcher):
    """Reference searcher that builds few-shot reference examples directly from the OHDSI Vocabulary
    tables plus a configured concept-vector table restricted to S terms in SNOMED.
    """

    def __init__(
        self,
        hierarchy_settings: HierarchySettings,
        exclude_concept_ids: set[int] | None = None,
        include_synonyms: bool = False,
    ):
        super().__init__(hierarchy_settings)
        self._exclude_concept_ids: list[int] = sorted(exclude_concept_ids) if exclude_concept_ids else []
        self._relationship_ids: list[str] = list(hierarchy_settings.snomed_relationships)
        self.include_synonyms = include_synonyms
        vector_settings = PgvectorSearchSettings(
            max_candidates=hierarchy_settings.retrieval.num_reference_examples * 2,
            filter=ConceptFilterSettings(
                standard_concept=["S"],
                vocabulary_ids=["SNOMED"],
            ),
            include_synonyms=include_synonyms,
            include_mapped_terms=True,
        )
        self._concept_searcher = PgvectorConceptSearcher(settings=vector_settings)

    def _search_reference_terms(self, text: str, top_k: int) -> list[tuple[int, str, float]]:
        fetch_limit = top_k * 2
        if self._concept_searcher.settings.max_candidates < fetch_limit:
            self._concept_searcher.settings.max_candidates = fetch_limit
        result_df = self._concept_searcher.search_term(term=text)
        if result_df is None or result_df.empty:
            return []

        if self._exclude_concept_ids:
            result_df = result_df[~result_df["concept_id"].isin(self._exclude_concept_ids)]
        if result_df.empty:
            return []

        top_df = result_df.head(top_k)
        # PgvectorConceptSearcher returns cosine distance; convert to similarity (higher = closer).
        return [
            (int(row.concept_id), str(row.concept_name), 1.0 - float(row.score))
            for row in top_df.itertuples(index=False)
        ]

    def _fetch_reference_attributes(self, source_concept_ids: list[int]) -> dict[int, list[dict]]:
        if not source_concept_ids:
            return {}

        query = sql.SQL("""
            SELECT cr.concept_id_1,
                   c2.concept_id,
                   c2.concept_code,
                   c2.concept_name,
                   cr.relationship_id
            FROM {schema}.concept_relationship cr
            INNER JOIN {schema}.concept c2 
                ON c2.concept_id = cr.concept_id_2
            WHERE cr.concept_id_1 = ANY(%s)
              AND cr.relationship_id = ANY(%s)
              AND cr.invalid_reason IS NULL
        """).format(schema=sql.Identifier(self.schema))

        with self.connection.cursor() as cur:
            cur.execute(query, (source_concept_ids, self._relationship_ids))
            rows = cur.fetchall()

        attrs_by_id: dict[int, list[dict]] = {}
        for src_cid, tgt_id, tgt_code, tgt_name, attribute_category in rows:
            attrs_by_id.setdefault(src_cid, []).append(
                {
                    "target_concept_id": tgt_id,
                    "target_concept_code": tgt_code,
                    "target_concept_name": tgt_name,
                    "attribute_category": attribute_category,
                }
            )
        return attrs_by_id

    def search(
        self,
        text: str,
        top_k: int | None = None,
    ) -> ReferenceSearchResult:
        top_terms = self._search_reference_terms(text, top_k=top_k)
        total_cost = self.get_total_cost()
        if not top_terms:
            return ReferenceSearchResult([], total_cost)

        attrs_by_id = self._fetch_reference_attributes([row[0] for row in top_terms])
        examples = [
            {
                "concept_id": concept_id_1,
                "concept_name": concept_name_1,
                "similarity": similarity,
                "attributes": attrs_by_id.get(concept_id_1, []),
            }
            for concept_id_1, concept_name_1, similarity in top_terms
        ]
        return ReferenceSearchResult(examples, total_cost)

    def get_total_cost(self) -> float:
        return self._cost + self._concept_searcher.get_total_cost()

    def close(self):
        self._concept_searcher.close()
        super().close()