Skip to content

hecate_concept_searcher

HecateConceptSearcher

Bases: AbstractConceptSearcher

A concept searcher that uses the OHDSI Hecate API to find concepts based on query strings.

Source code in src/ariadne/vector_search/hecate_concept_searcher.py
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
class HecateConceptSearcher(AbstractConceptSearcher):

    """
    A concept searcher that uses the OHDSI Hecate API to find concepts based on query strings.
    """

    def __init__(
        self,
        settings: HecateSearchSettings,
    ):
        """
        Initializes the HecateConceptSearcher.

        Args:
            settings: Vector search settings controlling endpoint selection,
                candidate limits, and query filters.
        """
        self.settings = settings
        self._sorted_substrings_to_remove = sorted(
            self.settings.substrings_to_remove,
            key=len,
            reverse=True,
        )
        self.default_url = _HECATE_SEARCH_STANDARD_URL
        self.default_params = {}
        standard_concepts = self.settings.filter.standard_concept
        uses_standard_only_endpoint = len(standard_concepts) == 1 and standard_concepts[0] == "S"
        if not uses_standard_only_endpoint:
            self.default_url = _HECATE_SEARCH_URL
            self.default_params["standard_concept"] = ",".join(standard_concepts)

        if self.settings.filter.domain_ids:
            self.default_params["domain_id"] = ",".join(self.settings.filter.domain_ids)
        if self.settings.filter.concept_class_ids:
            self.default_params["concept_class_id"] = ",".join(self.settings.filter.concept_class_ids)
        if self.settings.filter.vocabulary_ids:
            self.default_params["vocabulary_id"] = ",".join(self.settings.filter.vocabulary_ids)
        if self.settings.filter.exclude_vocabulary_ids:
            self.default_params["exclude_vocabulary_id"] = ",".join(self.settings.filter.exclude_vocabulary_ids)

    def search_term(
        self,
        query_string: str,
    ) -> Optional[pd.DataFrame]:
        """
        Searches for concepts matching the given query string.

        Args:
            query_string: The term to search for.

        Returns:
            A DataFrame containing the matching concepts, with the same columns as the concept table in the OMOP CDM,
            plus a 'score' column indicating the relevance score from the search.

        """
        if query_string == 'Kent Pharma (UK) Ltd':
            print("check")


        # Remove substrings from query_string
        cleaned_query = query_string
        for substring in self._sorted_substrings_to_remove:
            cleaned_query = re.sub(re.escape(substring), "", cleaned_query, flags=re.IGNORECASE)
        cleaned_query = cleaned_query.strip()

        endpoint = self.default_url
        params = dict(self.default_params)
        params.update({"q": cleaned_query, "limit": self.settings.max_candidates})
        response = None

        try:
            response = requests.get(endpoint, params=params, timeout=15)
            response.raise_for_status()
            terms = response.json()
            concepts = []
            for term in terms:
                # add score:
                for concept in term.get("concepts", []):
                    concept["score"] = term.get("score", None)
                concepts.extend(term.get("concepts", []))
            results_df = pd.DataFrame(concepts)
            if results_df.empty:
                return results_df
            if self.settings.filter.vocabulary_ids and "vocabulary_id" in results_df.columns:
                results_df = results_df[results_df["vocabulary_id"].isin(self.settings.filter.vocabulary_ids)]
            return results_df

        except requests.exceptions.HTTPError as http_err:
            print(f"HTTP error occurred: {http_err}")
            if response is not None:
                print(f"Response status code: {response.status_code}")
                print(f"Response content: {response.text}")
        except requests.exceptions.ConnectionError as conn_err:
            print(f"Connection error occurred: {conn_err}")
        except requests.exceptions.Timeout as timeout_err:
            print(f"The request timed out: {timeout_err}")
        except requests.exceptions.RequestException as err:
            print(f"An unexpected error occurred: {err}")

        return None

    def search_terms(
        self,
        df: pd.DataFrame,
        term_column: str,
        matched_concept_id_column: str = "matched_concept_id",
        matched_concept_name_column: str = "matched_concept_name",
        match_score_column: str = "match_score",
        match_rank_column: str = "match_rank",
    ) -> pd.DataFrame:
        """
        Searches the Hecate API for concepts matching terms in a DataFrame column.

        Args:
            df: DataFrame containing the terms to search for.
            term_column: Name of the column with terms to search.
            matched_concept_id_column: Name of the column to store matched concept IDs.
            matched_concept_name_column: Name of the column to store matched concept names.
            match_score_column: Name of the column to store match scores.
            match_rank_column: Name of the column to store match ranks.
        Returns:
            A DataFrame containing the same columns as the input dataframe plus the matching concepts for each term. For
            each term in the input dataframe, multiple rows will be returned corresponding to each matching concept.

        """

        all_results = []
        for index, row in df.iterrows():
            term = row[term_column]
            print(f"Processing term '{term}'")
            results = self.search_term(
                term,
            )
            if results is not None:
                rows = []
                for rank, (_, concept) in enumerate(results.iterrows(), start=1):
                    rows.append(
                        {
                            matched_concept_id_column: concept["concept_id"],
                            matched_concept_name_column: concept["concept_name"],
                            match_score_column: concept["score"],
                            match_rank_column: rank,
                        }
                    )
                results = pd.DataFrame(rows)
                results[match_rank_column] = range(1, len(results) + 1)
                orig_cols = list(df.columns)
                new_columns = list(results.columns)
                results[term_column] = term
                for col in df.columns:
                    results[col] = row[col]
                results = results[orig_cols + new_columns]
                if not results.empty:
                    all_results.append(results)

        if not all_results:
            return pd.DataFrame()

        return pd.concat(all_results, ignore_index=True)

__init__(settings)

Initializes the HecateConceptSearcher.

Parameters:

Name Type Description Default
settings HecateSearchSettings

Vector search settings controlling endpoint selection, candidate limits, and query filters.

required
Source code in src/ariadne/vector_search/hecate_concept_searcher.py
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def __init__(
    self,
    settings: HecateSearchSettings,
):
    """
    Initializes the HecateConceptSearcher.

    Args:
        settings: Vector search settings controlling endpoint selection,
            candidate limits, and query filters.
    """
    self.settings = settings
    self._sorted_substrings_to_remove = sorted(
        self.settings.substrings_to_remove,
        key=len,
        reverse=True,
    )
    self.default_url = _HECATE_SEARCH_STANDARD_URL
    self.default_params = {}
    standard_concepts = self.settings.filter.standard_concept
    uses_standard_only_endpoint = len(standard_concepts) == 1 and standard_concepts[0] == "S"
    if not uses_standard_only_endpoint:
        self.default_url = _HECATE_SEARCH_URL
        self.default_params["standard_concept"] = ",".join(standard_concepts)

    if self.settings.filter.domain_ids:
        self.default_params["domain_id"] = ",".join(self.settings.filter.domain_ids)
    if self.settings.filter.concept_class_ids:
        self.default_params["concept_class_id"] = ",".join(self.settings.filter.concept_class_ids)
    if self.settings.filter.vocabulary_ids:
        self.default_params["vocabulary_id"] = ",".join(self.settings.filter.vocabulary_ids)
    if self.settings.filter.exclude_vocabulary_ids:
        self.default_params["exclude_vocabulary_id"] = ",".join(self.settings.filter.exclude_vocabulary_ids)

search_term(query_string)

Searches for concepts matching the given query string.

Parameters:

Name Type Description Default
query_string str

The term to search for.

required

Returns:

Type Description
Optional[DataFrame]

A DataFrame containing the matching concepts, with the same columns as the concept table in the OMOP CDM,

Optional[DataFrame]

plus a 'score' column indicating the relevance score from the search.

Source code in src/ariadne/vector_search/hecate_concept_searcher.py
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def search_term(
    self,
    query_string: str,
) -> Optional[pd.DataFrame]:
    """
    Searches for concepts matching the given query string.

    Args:
        query_string: The term to search for.

    Returns:
        A DataFrame containing the matching concepts, with the same columns as the concept table in the OMOP CDM,
        plus a 'score' column indicating the relevance score from the search.

    """
    if query_string == 'Kent Pharma (UK) Ltd':
        print("check")


    # Remove substrings from query_string
    cleaned_query = query_string
    for substring in self._sorted_substrings_to_remove:
        cleaned_query = re.sub(re.escape(substring), "", cleaned_query, flags=re.IGNORECASE)
    cleaned_query = cleaned_query.strip()

    endpoint = self.default_url
    params = dict(self.default_params)
    params.update({"q": cleaned_query, "limit": self.settings.max_candidates})
    response = None

    try:
        response = requests.get(endpoint, params=params, timeout=15)
        response.raise_for_status()
        terms = response.json()
        concepts = []
        for term in terms:
            # add score:
            for concept in term.get("concepts", []):
                concept["score"] = term.get("score", None)
            concepts.extend(term.get("concepts", []))
        results_df = pd.DataFrame(concepts)
        if results_df.empty:
            return results_df
        if self.settings.filter.vocabulary_ids and "vocabulary_id" in results_df.columns:
            results_df = results_df[results_df["vocabulary_id"].isin(self.settings.filter.vocabulary_ids)]
        return results_df

    except requests.exceptions.HTTPError as http_err:
        print(f"HTTP error occurred: {http_err}")
        if response is not None:
            print(f"Response status code: {response.status_code}")
            print(f"Response content: {response.text}")
    except requests.exceptions.ConnectionError as conn_err:
        print(f"Connection error occurred: {conn_err}")
    except requests.exceptions.Timeout as timeout_err:
        print(f"The request timed out: {timeout_err}")
    except requests.exceptions.RequestException as err:
        print(f"An unexpected error occurred: {err}")

    return None

search_terms(df, term_column, matched_concept_id_column='matched_concept_id', matched_concept_name_column='matched_concept_name', match_score_column='match_score', match_rank_column='match_rank')

Searches the Hecate API for concepts matching terms in a DataFrame column.

Parameters:

Name Type Description Default
df DataFrame

DataFrame containing the terms to search for.

required
term_column str

Name of the column with terms to search.

required
matched_concept_id_column str

Name of the column to store matched concept IDs.

'matched_concept_id'
matched_concept_name_column str

Name of the column to store matched concept names.

'matched_concept_name'
match_score_column str

Name of the column to store match scores.

'match_score'
match_rank_column str

Name of the column to store match ranks.

'match_rank'

Returns: A DataFrame containing the same columns as the input dataframe plus the matching concepts for each term. For each term in the input dataframe, multiple rows will be returned corresponding to each matching concept.

Source code in src/ariadne/vector_search/hecate_concept_searcher.py
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
def search_terms(
    self,
    df: pd.DataFrame,
    term_column: str,
    matched_concept_id_column: str = "matched_concept_id",
    matched_concept_name_column: str = "matched_concept_name",
    match_score_column: str = "match_score",
    match_rank_column: str = "match_rank",
) -> pd.DataFrame:
    """
    Searches the Hecate API for concepts matching terms in a DataFrame column.

    Args:
        df: DataFrame containing the terms to search for.
        term_column: Name of the column with terms to search.
        matched_concept_id_column: Name of the column to store matched concept IDs.
        matched_concept_name_column: Name of the column to store matched concept names.
        match_score_column: Name of the column to store match scores.
        match_rank_column: Name of the column to store match ranks.
    Returns:
        A DataFrame containing the same columns as the input dataframe plus the matching concepts for each term. For
        each term in the input dataframe, multiple rows will be returned corresponding to each matching concept.

    """

    all_results = []
    for index, row in df.iterrows():
        term = row[term_column]
        print(f"Processing term '{term}'")
        results = self.search_term(
            term,
        )
        if results is not None:
            rows = []
            for rank, (_, concept) in enumerate(results.iterrows(), start=1):
                rows.append(
                    {
                        matched_concept_id_column: concept["concept_id"],
                        matched_concept_name_column: concept["concept_name"],
                        match_score_column: concept["score"],
                        match_rank_column: rank,
                    }
                )
            results = pd.DataFrame(rows)
            results[match_rank_column] = range(1, len(results) + 1)
            orig_cols = list(df.columns)
            new_columns = list(results.columns)
            results[term_column] = term
            for col in df.columns:
                results[col] = row[col]
            results = results[orig_cols + new_columns]
            if not results.empty:
                all_results.append(results)

    if not all_results:
        return pd.DataFrame()

    return pd.concat(all_results, ignore_index=True)