Skip to content

term_cleaner

TermCleaner

A class to clean clinical terms by removing non-essential modifiers and information using a Large Language Model (LLM).

Source code in src/ariadne/term_cleanup/term_cleaner.py
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
class TermCleaner:
    """
    A class to clean clinical terms by removing non-essential modifiers and information using a Large Language Model (LLM).
    """

    def __init__(self, settings: TermCleanerSettings):
        self.system_prompt = settings.system_prompt
        self.max_workers = settings.max_workers
        self.cost = 0.0
        self._cost_lock = threading.Lock()

        and_or = settings.and_or_rewrite
        self._and_or_vocabularies = frozenset(and_or.vocabulary_ids)
        self._and_or_excluded_codes = frozenset(and_or.excluded_codes)
        self._and_or_excluded_prefixes = tuple(and_or.excluded_prefixes)
        self._and_or_both_combined_exception_prefixes = tuple(and_or.both_combined_exception_prefixes)
        self._and_or_excluded_name_re = (
            re.compile("|".join(and_or.excluded_name_patterns), flags=re.IGNORECASE)
            if and_or.excluded_name_patterns
            else None
        )

    def _should_replace_and(self, term: str, vocabulary_id: str, concept_code: str) -> bool:
        """
        Return True if every ' and ' in *term* should be rewritten to ' and/or '.

        Rules (data supplied via AndOrRewriteSettings):
          - vocabulary_id must be in the configured vocabularies
          - concept_code must NOT be an excluded exact code or have an excluded prefix
          - term must NOT contain 'both'/'combined' (unless code has a both/combined exception prefix)
          - term must NOT match any TRUE_AND name pattern
        """
        if vocabulary_id not in self._and_or_vocabularies:
            return False
        if not re.search(r" and ", term, flags=re.IGNORECASE):
            return False
        if concept_code in self._and_or_excluded_codes:
            return False
        if any(concept_code.startswith(p) for p in self._and_or_excluded_prefixes):
            return False
        if re.search(r"\b(both|combined)\b", term, flags=re.IGNORECASE):
            if not any(concept_code.startswith(p) for p in self._and_or_both_combined_exception_prefixes):
                return False
        if self._and_or_excluded_name_re is not None and self._and_or_excluded_name_re.search(term):
            return False
        return True

    def rewrite_and(
        self, term: str, vocabulary_id: str = "", concept_code: str = ""
    ) -> str:
        """
        Rewrite ' and ' → ' and/or ' for ICD-family terms where appropriate.
        Args:
            term: The clinical term to be cleaned.
            vocabulary_id: Source vocabulary (e.g. 'ICD10CM'). Used for
                           the and/or rewrite; safe to omit for non-ICD terms.
            concept_code: Source concept code. Used for and/or rewrite
                          exclusions; safe to omit.

        Returns:
            The cleaned clinical term.
        """
        if self._should_replace_and(term, vocabulary_id, concept_code):
            term = re.sub(r" and ", " and/or ", term, flags=re.IGNORECASE)
        return term

    def _clean_terms_batch(self, terms: list[str]) -> list[str]:
        """Cleans up to 25 terms in one LLM request and returns results in input order."""
        if not terms:
            return []

        rows = [{"row_number": i, "source_term": term} for i, term in enumerate(terms)]
        prompt = (
            "Input JSON:\n"
            f"{json.dumps({'terms': rows}, ensure_ascii=False)}"
        )
        response = get_llm_response(
            prompt=prompt,
            system_prompt=self.system_prompt,
            json_schema=_TERM_CLEANING_BATCH_SCHEMA,
            json_schema_name="term_cleaning_batch",
        )
        with self._cost_lock:
            self.cost += response["usage"]["total_cost_usd"]
        cleaned_terms = list(terms)
        parsed = response["parsed_json"]
        if not isinstance(parsed, dict):
            raise ValueError("Term cleaning response must be a JSON object with a 'results' array.")
        results = parsed["results"]
        if not isinstance(results, list):
            raise ValueError("Term cleaning response 'results' must be a list.")
        for item in results:
            if not isinstance(item, dict):
                raise ValueError("Each term cleaning result must be an object.")
            if "row_number" not in item:
                raise ValueError("Each term cleaning result must include integer 'row_number'.")
            if "cleaned_term" not in item:
                raise ValueError("Each term cleaning result must include string 'cleaned_term'.")
            row_number = item["row_number"]
            cleaned_term = item["cleaned_term"]
            if not isinstance(row_number, int):
                raise ValueError("Each term cleaning result must include integer 'row_number'.")
            if row_number < 0 or row_number >= len(cleaned_terms):
                raise ValueError("Row number out of range in term cleaning result.")
            if not isinstance(cleaned_term, str):
                raise ValueError("Each term cleaning result must include string 'cleaned_term'.")
            cleaned_term = cleaned_term.strip()
            cleaned_terms[row_number] = cleaned_term
        return cleaned_terms

    def clean_terms(
        self,
        df: pd.DataFrame,
        term_column: str = "source_term",
        output_column: str = "cleaned_term",
        vocabulary_column: str = "source_vocabulary_id",
        code_column: str = "source_concept_code",
    ) -> pd.DataFrame:
        """
        Cleans clinical terms in a DataFrame column.

        LLM cleanup runs first, then the and/or rewrite is applied to the cleaned
        terms.  Running the rewrite last keeps it deterministic, since the LLM
        cannot collapse ' and/or ' back to ' and '.

        *term_column*, *vocabulary_column*, and *code_column* are all required and
        must be present in *df*; a ValueError is raised otherwise.

        Args:
            df: DataFrame containing the terms to be cleaned.
            term_column: Column with source terms.
            output_column: Column to write cleaned terms to.
            vocabulary_column: Column with the source vocabulary id.
            code_column: Column with the source concept code.

        Returns:
            DataFrame with cleaned terms in *output_column*.
        """
        missing_columns = [c for c in (term_column, vocabulary_column, code_column) if c not in df.columns]
        if missing_columns:
            raise ValueError(f"clean_terms requires the following missing columns: {missing_columns}")

        terms = df[term_column].astype(str).tolist()

        # Batches are sent concurrently (settings.max_workers); results are written
        # back to *df* here, in the calling thread, keyed by each batch's row index.
        batch_starts = list(range(0, len(terms), _BATCH_SIZE))
        batches = [terms[start : start + _BATCH_SIZE] for start in batch_starts]
        for batch_number, cleaned_batch in run_concurrently(batches, self._clean_terms_batch, self.max_workers):
            start = batch_starts[batch_number]
            df.loc[df.index[start : start + _BATCH_SIZE], output_column] = cleaned_batch

        vocab_values = df[vocabulary_column].fillna("").astype(str).tolist()
        code_values = df[code_column].fillna("").astype(str).tolist()
        cleaned_values = df[output_column].astype(str).tolist()
        df[output_column] = [
            self.rewrite_and(cleaned_term, vocabulary_id, concept_code)
            for cleaned_term, vocabulary_id, concept_code in zip(cleaned_values, vocab_values, code_values)
        ]

        # Remove rows where LLM returned an empty cleaned term (sign of bad input like 'number of goats', 'declined' or 'primary')
        df = df[df[output_column].notna() & (df[output_column].str.strip() != "")].reset_index(drop=True)

        return df

    def get_total_cost(self) -> float:
        """
        Returns the total cost incurred for LLM calls during term cleaning.

        Returns:
            Total cost in USD.
        """

        return self.cost

clean_terms(df, term_column='source_term', output_column='cleaned_term', vocabulary_column='source_vocabulary_id', code_column='source_concept_code')

Cleans clinical terms in a DataFrame column.

LLM cleanup runs first, then the and/or rewrite is applied to the cleaned terms. Running the rewrite last keeps it deterministic, since the LLM cannot collapse ' and/or ' back to ' and '.

term_column, vocabulary_column, and code_column are all required and must be present in df; a ValueError is raised otherwise.

Parameters:

Name Type Description Default
df DataFrame

DataFrame containing the terms to be cleaned.

required
term_column str

Column with source terms.

'source_term'
output_column str

Column to write cleaned terms to.

'cleaned_term'
vocabulary_column str

Column with the source vocabulary id.

'source_vocabulary_id'
code_column str

Column with the source concept code.

'source_concept_code'

Returns:

Type Description
DataFrame

DataFrame with cleaned terms in output_column.

Source code in src/ariadne/term_cleanup/term_cleaner.py
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
def clean_terms(
    self,
    df: pd.DataFrame,
    term_column: str = "source_term",
    output_column: str = "cleaned_term",
    vocabulary_column: str = "source_vocabulary_id",
    code_column: str = "source_concept_code",
) -> pd.DataFrame:
    """
    Cleans clinical terms in a DataFrame column.

    LLM cleanup runs first, then the and/or rewrite is applied to the cleaned
    terms.  Running the rewrite last keeps it deterministic, since the LLM
    cannot collapse ' and/or ' back to ' and '.

    *term_column*, *vocabulary_column*, and *code_column* are all required and
    must be present in *df*; a ValueError is raised otherwise.

    Args:
        df: DataFrame containing the terms to be cleaned.
        term_column: Column with source terms.
        output_column: Column to write cleaned terms to.
        vocabulary_column: Column with the source vocabulary id.
        code_column: Column with the source concept code.

    Returns:
        DataFrame with cleaned terms in *output_column*.
    """
    missing_columns = [c for c in (term_column, vocabulary_column, code_column) if c not in df.columns]
    if missing_columns:
        raise ValueError(f"clean_terms requires the following missing columns: {missing_columns}")

    terms = df[term_column].astype(str).tolist()

    # Batches are sent concurrently (settings.max_workers); results are written
    # back to *df* here, in the calling thread, keyed by each batch's row index.
    batch_starts = list(range(0, len(terms), _BATCH_SIZE))
    batches = [terms[start : start + _BATCH_SIZE] for start in batch_starts]
    for batch_number, cleaned_batch in run_concurrently(batches, self._clean_terms_batch, self.max_workers):
        start = batch_starts[batch_number]
        df.loc[df.index[start : start + _BATCH_SIZE], output_column] = cleaned_batch

    vocab_values = df[vocabulary_column].fillna("").astype(str).tolist()
    code_values = df[code_column].fillna("").astype(str).tolist()
    cleaned_values = df[output_column].astype(str).tolist()
    df[output_column] = [
        self.rewrite_and(cleaned_term, vocabulary_id, concept_code)
        for cleaned_term, vocabulary_id, concept_code in zip(cleaned_values, vocab_values, code_values)
    ]

    # Remove rows where LLM returned an empty cleaned term (sign of bad input like 'number of goats', 'declined' or 'primary')
    df = df[df[output_column].notna() & (df[output_column].str.strip() != "")].reset_index(drop=True)

    return df

get_total_cost()

Returns the total cost incurred for LLM calls during term cleaning.

Returns:

Type Description
float

Total cost in USD.

Source code in src/ariadne/term_cleanup/term_cleaner.py
221
222
223
224
225
226
227
228
229
def get_total_cost(self) -> float:
    """
    Returns the total cost incurred for LLM calls during term cleaning.

    Returns:
        Total cost in USD.
    """

    return self.cost

rewrite_and(term, vocabulary_id='', concept_code='')

Rewrite ' and ' → ' and/or ' for ICD-family terms where appropriate. Args: term: The clinical term to be cleaned. vocabulary_id: Source vocabulary (e.g. 'ICD10CM'). Used for the and/or rewrite; safe to omit for non-ICD terms. concept_code: Source concept code. Used for and/or rewrite exclusions; safe to omit.

Returns:

Type Description
str

The cleaned clinical term.

Source code in src/ariadne/term_cleanup/term_cleaner.py
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def rewrite_and(
    self, term: str, vocabulary_id: str = "", concept_code: str = ""
) -> str:
    """
    Rewrite ' and ' → ' and/or ' for ICD-family terms where appropriate.
    Args:
        term: The clinical term to be cleaned.
        vocabulary_id: Source vocabulary (e.g. 'ICD10CM'). Used for
                       the and/or rewrite; safe to omit for non-ICD terms.
        concept_code: Source concept code. Used for and/or rewrite
                      exclusions; safe to omit.

    Returns:
        The cleaned clinical term.
    """
    if self._should_replace_and(term, vocabulary_id, concept_code):
        term = re.sub(r" and ", " and/or ", term, flags=re.IGNORECASE)
    return term