Skip to content

hierarchy_attribute_evaluator

Evaluation utilities for the SNOMED CT attribute extraction pipeline.

Public API

build_prediction_rows — flatten pipeline output into row-wise predictions. evaluate_results — full outer join evaluation producing P/R/F1.

build_prediction_rows(results, hierarchy_settings)

Extract prediction rows from pipeline results for evaluation.

Handles regular attributes (single dict, list of dicts) and paired interprets_interpretation structures via :func:split_interprets_pairs. Each row carries relationship_groups: the SNOMED role group number(s) of the attribute value, ;-separated (a value shared by several groups appears once).

Parameters:

Name Type Description Default
results list[dict]

Pipeline result dicts from process_hierarchy.

required
hierarchy_settings HierarchySettings

Pipeline configuration. hierarchy.attributes gives each attribute key's OHDSI relationship (attribute_category). When hierarchy.mrcm.grouping is configured, results without role_groups (e.g. cached before grouping existed) are grouped on the fly; otherwise their relationship_groups is None.

required

Returns:

Type Description
list[dict]

List of flat dicts ready for pd.DataFrame.

Source code in src/ariadne/evaluation/hierarchy_attribute_evaluator.py
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def build_prediction_rows(results: list[dict], hierarchy_settings: HierarchySettings) -> list[dict]:
    """Extract prediction rows from pipeline results for evaluation.

    Handles regular attributes (single dict, list of dicts) and paired
    ``interprets_interpretation`` structures via :func:`split_interprets_pairs`.
    Each row carries ``relationship_groups``: the SNOMED role group number(s) of the
    attribute value, ``;``-separated (a value shared by several groups appears once).

    Args:
        results: Pipeline result dicts from ``process_hierarchy``.
        hierarchy_settings: Pipeline configuration. ``hierarchy.attributes`` gives each attribute
            key's OHDSI relationship (``attribute_category``). When ``hierarchy.mrcm.grouping`` is
            configured, results without ``role_groups`` (e.g. cached before grouping existed) are
            grouped on the fly; otherwise their ``relationship_groups`` is ``None``.

    Returns:
        List of flat dicts ready for ``pd.DataFrame``.
    """
    relationship_for = hierarchy_settings.relationship_by_attribute()
    mrcm_settings = hierarchy_settings.mrcm

    def _category(attr_key: str, default: str) -> str:
        return relationship_for.get(EXTRACTION_KEY_ALIASES.get(attr_key, attr_key), default)

    pred_rows: list[dict] = []
    for result in results:
        source_meta = {
            "source_key": result.get("source_key"),
            "source_concept_id": result.get("source_concept_id"),
            "source_concept_code": result.get("source_concept_code"),
            "source_vocabulary_id": result.get("source_vocabulary_id"),
            "source_term": result.get("source_term"),
        }
        if "attributes" not in result:
            continue
        role_groups = result.get("role_groups")
        if role_groups is None and mrcm_settings.grouping.enabled:
            role_groups = group_attributes(
                result["attributes"], mrcm_settings, result.get("llm_role_groups")
            ).role_groups
        group_lookup = _role_group_lookup(role_groups)

        def _groups(key: str, concept: dict) -> str | None:
            return group_lookup.get((key, str(concept.get("concept_id"))))
        for attr_key, attr_value in result["attributes"].items():
            if attr_value is None:
                continue

            # Handle paired interprets_interpretation tuples
            if attr_key == "interprets_interpretation":
                if not isinstance(attr_value, list):
                    attr_value = [attr_value]
                for sub_key, concept in split_interprets_pairs(attr_value):
                    if isinstance(concept, dict):
                        pred_rows.append({
                            **source_meta,
                            "predicted_target_concept_id": concept.get("concept_id"),
                            "predicted_target_concept_name": concept.get("concept_name"),
                            "predicted_target_concept_code": concept.get("concept_code"),
                            "attribute_category": _category(sub_key, f"Has {sub_key}"),
                            "relationship_groups": _groups(sub_key, concept),
                        })
                continue

            attr_type = _category(attr_key, attr_key)
            # Handle list of concept dicts (new multi-value format)
            if isinstance(attr_value, list):
                for item in attr_value:
                    if item and isinstance(item, dict):
                        pred_rows.append({
                            **source_meta,
                            "predicted_target_concept_id": item.get("concept_id"),
                            "predicted_target_concept_name": item.get("concept_name"),
                            "predicted_target_concept_code": item.get("concept_code"),
                            "attribute_category": attr_type,
                            "relationship_groups": _groups(attr_key, item),
                        })
            # Handle single concept dict
            elif isinstance(attr_value, dict):
                pred_rows.append({
                    **source_meta,
                    "predicted_target_concept_id": attr_value.get("concept_id"),
                    "predicted_target_concept_name": attr_value.get("concept_name"),
                    "predicted_target_concept_code": attr_value.get("concept_code"),
                    "attribute_category": attr_type,
                    "relationship_groups": _groups(attr_key, attr_value),
                })
    return pred_rows

evaluate_results(results, gs_path, output_dir, hierarchy_settings)

Produce a combined evaluation table (full outer join of GS and predictions).

Columns

source_key, source_concept_id, source_concept_code, source_vocabulary_id, source_term, attribute_category, gs_target_concept_id, gs_target_concept_code, gs_target_concept_name, predicted_target_concept_id, predicted_target_concept_code, predicted_target_concept_name, matched, status (match / missed / extra).

Summary statistics are printed and the combined table is saved to CSV.

Parameters:

Name Type Description Default
results list[dict]

List of pipeline result dicts from process_hierarchy.

required
gs_path str

Path to the gold-standard CSV.

required
output_dir Path

Path wherhe the output CSV should be saved.

required
hierarchy_settings HierarchySettings

Pipeline configuration (see :func:build_prediction_rows).

required

Returns:

Type Description
DataFrame

Combined evaluation DataFrame.

Source code in src/ariadne/evaluation/hierarchy_attribute_evaluator.py
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
def evaluate_results(
    results: list[dict],
    gs_path: str,
    output_dir: Path,
    hierarchy_settings: HierarchySettings,
) -> pd.DataFrame:
    """Produce a combined evaluation table (full outer join of GS and predictions).

    Columns:
        source_key, source_concept_id, source_concept_code, source_vocabulary_id,
        source_term, attribute_category,
        gs_target_concept_id, gs_target_concept_code, gs_target_concept_name,
        predicted_target_concept_id, predicted_target_concept_code, predicted_target_concept_name,
        matched, status (``match`` / ``missed`` / ``extra``).

    Summary statistics are printed and the combined table is saved to CSV.

    Args:
        results: List of pipeline result dicts from ``process_hierarchy``.
        gs_path: Path to the gold-standard CSV.
        output_dir: Path wherhe the output CSV should be saved.
        hierarchy_settings: Pipeline configuration (see :func:`build_prediction_rows`).

    Returns:
        Combined evaluation DataFrame.
    """
    # --- build predicted rows ---
    pred_rows = build_prediction_rows(results, hierarchy_settings)
    pred_df = pd.DataFrame(pred_rows)
    source_cols = ["source_key", "source_concept_id", "source_concept_code",
                   "source_vocabulary_id", "source_term"]
    if pred_df.empty:
        pred_df = pd.DataFrame(columns=source_cols + [
            "predicted_target_concept_id", "predicted_target_concept_name",
            "predicted_target_concept_code", "attribute_category", "relationship_groups"])

    # --- load gold standard ---
    gs_df = pd.read_csv(gs_path)
    gs_df = gs_df.rename(columns={
        'target_concept_id': 'gs_target_concept_id',
        'target_concept_code': 'gs_target_concept_code',
        'target_concept_name': 'gs_target_concept_name',
    })

    # --- composite source key + numeric target join key ---
    for df in (gs_df, pred_df):
        df['source_key'] = (
            df['source_vocabulary_id'].astype(str) + ':' + df['source_concept_code'].astype(str)
        )
    gs_df['_join_target'] = pd.to_numeric(gs_df['gs_target_concept_id'], errors='coerce').astype('Int64')
    pred_df['_join_target'] = pd.to_numeric(pred_df['predicted_target_concept_id'], errors='coerce').astype('Int64')

    combined = gs_df.merge(
        pred_df,
        on=['source_key', '_join_target', 'attribute_category'],
        how='outer',
        suffixes=('_gs', '_pred'),
    )

    # Reconcile source descriptor columns from both sides (GS wins, pred fills gaps)
    for col in ('source_concept_id', 'source_concept_code', 'source_vocabulary_id', 'source_term'):
        gs_col, pred_col = f'{col}_gs', f'{col}_pred'
        if gs_col in combined.columns:
            combined[col] = combined[gs_col].fillna(combined[pred_col])
            combined.drop(columns=[gs_col, pred_col], inplace=True)

    combined.drop(columns=['_join_target'], inplace=True)

    # --- flags ---
    has_gs = combined['gs_target_concept_id'].notna()
    has_pred = combined['predicted_target_concept_id'].notna()
    combined['matched'] = has_gs & has_pred
    combined['status'] = 'match'
    combined.loc[has_gs & ~has_pred, 'status'] = 'missed'
    combined.loc[~has_gs & has_pred, 'status'] = 'extra'

    # --- order columns nicely ---
    leading = ['source_key', 'source_concept_id', 'source_concept_code',
               'source_vocabulary_id', 'source_term', 'attribute_category',
               'gs_target_concept_id', 'gs_target_concept_code', 'gs_target_concept_name',
               'predicted_target_concept_id', 'predicted_target_concept_code', 'predicted_target_concept_name',
               'matched', 'status']
    extra_cols = [c for c in combined.columns if c not in leading]
    combined = combined[[c for c in leading if c in combined.columns] + extra_cols]

    # Sort for readability
    combined = combined.sort_values(['source_key', 'attribute_category', 'status']).reset_index(drop=True)

    # --- summary stats ---
    n_gs = int(has_gs.sum())
    n_pred = int(has_pred.sum())
    n_match = int(combined['matched'].sum())
    precision = n_match / n_pred * 100 if n_pred else 0.0
    recall = n_match / n_gs * 100 if n_gs else 0.0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0

    logger.info("Gold standard rows: %d", n_gs)
    logger.info("Predicted rows:     %d", n_pred)
    logger.info("Matched:            %d", n_match)
    logger.info("Precision:          %.1f%%", precision)
    logger.info("Recall:             %.1f%%", recall)
    logger.info("F1:                 %.1f%%", f1)

    # --- save ---
    out_path = os.path.join(output_dir, "hierarchy_attribute_evaluation.csv")
    combined.to_csv(out_path, index=False)
    logger.info("Combined evaluation saved: %s (%d rows)", out_path, len(combined))
    return combined