Skip to content

llm_attribute_runner

Batch execution helpers for hierarchy attribute extraction.

The module exposes a single public entry point, :func:process_hierarchy, which executes the term-level extraction pipeline over a tabular dataset. Execution is resumable: each term's result is written to its own JSON file as soon as it succeeds, so an interrupted run resumes by reprocessing only the terms without a cached file. Failed terms are left uncached and retried on the next run.

Terms are processed concurrently by max_workers threads (the work is dominated by waiting on LLM API calls). Workers only run the extraction; caching, cost tracking and logging happen in the calling thread.

load_hierarchy_results(path)

Load pipeline results from a JSON list file or a folder of per-term JSON files.

Unreadable per-term files and results without a source_key are skipped with a warning.

Source code in src/ariadne/hierarchy/llm_attribute_runner.py
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def load_hierarchy_results(path: str | Path) -> list[dict]:
    """Load pipeline results from a JSON list file or a folder of per-term JSON files.

    Unreadable per-term files and results without a ``source_key`` are skipped with a warning.
    """
    path = Path(path)
    if path.is_file():
        with open(path, encoding="utf-8") as f:
            data = json.load(f)
        loaded = data if isinstance(data, list) else [data]
    else:
        loaded = []
        for file in sorted(path.glob("*.json")):
            try:
                with open(file, encoding="utf-8") as f:
                    loaded.append(json.load(f))
            except (json.JSONDecodeError, OSError) as exc:
                logger.warning("Skipping unreadable result file %s: %s", file, exc)
    results = [r for r in loaded if isinstance(r, dict) and r.get("source_key")]
    if len(results) < len(loaded):
        logger.warning("Skipped %d result(s) without a source_key in %s", len(loaded) - len(results), path)
    return results

process_hierarchy(terms, attribute_searcher, reference_searcher, hierarchy_settings, source_term_column='source_term', source_code_column='source_concept_code', source_vocab_column='source_vocabulary_id', source_id_column='source_concept_id', checkpoint_every=5, max_workers=None)

Execute hierarchy extraction over source terms with checkpoint resume.

Parameters

terms Input rows containing source identifiers and term text. attribute_searcher Searcher used by the extraction pipeline for attribute candidates. reference_searcher Searcher used for optional reference concept lookups. hierarchy_settings Required runtime settings. evaluation.llm_response_folder is used to store one result file per source term for resume. source_term_column Name of the column containing source term text. source_code_column Name of the column containing the source concept code. source_vocab_column Name of the column containing the source vocabulary id. Combined with source_code_column to form the composite primary key. source_id_column Optional column with the OMOP concept_id (may be absent for new vocabularies). Carried through when present. checkpoint_every Cadence (in terms) for emitting a progress log line. Results are persisted immediately per term regardless of this value. max_workers Number of terms processed concurrently. Defaults to hierarchy_settings.evaluation.max_workers; 1 runs sequentially.

Returns

list[dict] One result dictionary per processed source code: previously cached results first, then newly processed terms in input order.

Raises

ValueError If hierarchy_settings is missing.

Source code in src/ariadne/hierarchy/llm_attribute_runner.py
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def process_hierarchy(
    terms: pd.DataFrame,
    attribute_searcher: AttributeSearcher,
    reference_searcher: ReferenceSearcher,
    hierarchy_settings: HierarchySettings | None,
    source_term_column: str = "source_term",
    source_code_column: str = "source_concept_code",
    source_vocab_column: str = "source_vocabulary_id",
    source_id_column: str | None = "source_concept_id",
    checkpoint_every: int = 5,
    max_workers: int | None = None,
) -> list[dict]:
    """Execute hierarchy extraction over source terms with checkpoint resume.

    Parameters
    ----------
    terms
        Input rows containing source identifiers and term text.
    attribute_searcher
        Searcher used by the extraction pipeline for attribute candidates.
    reference_searcher
        Searcher used for optional reference concept lookups.
    hierarchy_settings
        Required runtime settings. ``evaluation.llm_response_folder`` is used to
        store one result file per source term for resume.
    source_term_column
        Name of the column containing source term text.
    source_code_column
        Name of the column containing the source concept code.
    source_vocab_column
        Name of the column containing the source vocabulary id. Combined with
        ``source_code_column`` to form the composite primary key.
    source_id_column
        Optional column with the OMOP ``concept_id`` (may be absent for new
        vocabularies). Carried through when present.
    checkpoint_every
        Cadence (in terms) for emitting a progress log line. Results are
        persisted immediately per term regardless of this value.
    max_workers
        Number of terms processed concurrently. Defaults to
        ``hierarchy_settings.evaluation.max_workers``; ``1`` runs sequentially.

    Returns
    -------
    list[dict]
        One result dictionary per processed source code: previously cached
        results first, then newly processed terms in input order.

    Raises
    ------
    ValueError
        If ``hierarchy_settings`` is missing.
    """
    if not hierarchy_settings:
        raise ValueError("HierarchySettings must be provided")

    results_dir = Path(hierarchy_settings.evaluation.llm_response_folder)

    all_results = load_hierarchy_results(results_dir)
    processed_ids = {r["source_key"] for r in all_results}
    if all_results:
        logger.info(
            "Resuming: %d terms already cached in %s", len(all_results), results_dir
        )
        _drop_cached_non_candidate_values(all_results, hierarchy_settings)

    has_source_id = bool(source_id_column) and source_id_column in terms.columns

    def _row_source_key(row) -> str:
        return _make_source_key(
            getattr(row, source_vocab_column), getattr(row, source_code_column)
        )

    pending_terms = [
        row for row in terms.itertuples(index=False) if _row_source_key(row) not in processed_ids
    ]

    if not pending_terms:
        logger.info("All terms already processed.")
        return all_results

    total_cost = sum(r.get("cost", {}).get("total_cost", 0.0) for r in all_results)
    if max_workers is None:
        max_workers = getattr(hierarchy_settings.evaluation, "max_workers", 1)

    def _source_meta(row) -> dict:
        source_concept_code = str(getattr(row, source_code_column))
        source_vocabulary_id = str(getattr(row, source_vocab_column))
        raw_id = getattr(row, source_id_column) if has_source_id else None
        return {
            "source_key": _make_source_key(source_vocabulary_id, source_concept_code),
            "source_concept_id": None if raw_id is None or pd.isna(raw_id) else raw_id,
            "source_concept_code": source_concept_code,
            "source_vocabulary_id": source_vocabulary_id,
            "source_term": str(getattr(row, source_term_column)),
        }

    def _worker(source_meta: dict) -> dict:
        logger.debug("\n%s\n%s", "=" * 60, source_meta["source_term"])
        return _process_term(
            source_meta,
            source_meta["source_term"],
            attribute_searcher,
            reference_searcher,
            hierarchy_settings,
        )

    pending_meta = [_source_meta(row) for row in pending_terms]
    new_results: list[dict | None] = [None] * len(pending_meta)
    logger.info(
        "Processing %d pending terms with %d worker(s)", len(pending_meta), max(1, max_workers)
    )

    for idx, result in run_concurrently(pending_meta, _worker, max_workers):
        new_results[idx] = result
        source_key = result["source_key"]

        # Do not cache failures: leaving them unpersisted means they are retried
        # on the next run (mirrors the LlmMapper per-response file cache).
        if "error" in result:
            logger.warning(
                "Term %s failed; not caching so it retries next run.", source_key
            )
            continue

        _save_result(results_dir, result)
        processed_ids.add(source_key)
        if "cost" in result:
            total_cost += result["cost"]["total_cost"]
            logger.debug(
                "  %s cost: $%.4f | running total: $%.4f",
                source_key,
                result["cost"]["total_cost"],
                total_cost,
            )

        if checkpoint_every > 0 and len(processed_ids) % checkpoint_every == 0:
            logger.info(
                "Processed %d/%d codes | Total API cost: $%.4f",
                len(processed_ids),
                len(terms),
                total_cost,
            )

    all_results.extend(r for r in new_results if r is not None)

    logger.debug(
        "\n%s\nCompleted: %d terms, Total cost: $%.4f",
        "=" * 60,
        len(all_results),
        total_cost,
    )

    return all_results