Spaces:
Running
Running
petrsovadina
commited on
Commit
•
da5b905
1
Parent(s):
fb60317
Update presidio_nlp_engine_config.py
Browse files
presidio_nlp_engine_config.py
CHANGED
@@ -48,32 +48,7 @@ def create_nlp_engine_with_spacy(
|
|
48 |
return nlp_engine, registry
|
49 |
|
50 |
|
51 |
-
|
52 |
-
model_path: str,
|
53 |
-
) -> Tuple[NlpEngine, RecognizerRegistry]:
|
54 |
-
"""
|
55 |
-
Instantiate an NlpEngine with a stanza model
|
56 |
-
:param model_path: path to model / model name.
|
57 |
-
"""
|
58 |
-
nlp_configuration = {
|
59 |
-
"nlp_engine_name": "stanza",
|
60 |
-
"models": [{"lang_code": "en", "model_name": model_path}],
|
61 |
-
"ner_model_configuration": {
|
62 |
-
"model_to_presidio_entity_mapping": {
|
63 |
-
"PER": "PERSON",
|
64 |
-
"PERSON": "PERSON",
|
65 |
-
"NORP": "NRP",
|
66 |
-
"FAC": "FACILITY",
|
67 |
-
"LOC": "LOCATION",
|
68 |
-
"GPE": "LOCATION",
|
69 |
-
"LOCATION": "LOCATION",
|
70 |
-
"ORG": "ORGANIZATION",
|
71 |
-
"ORGANIZATION": "ORGANIZATION",
|
72 |
-
"DATE": "DATE_TIME",
|
73 |
-
"TIME": "DATE_TIME",
|
74 |
-
}
|
75 |
-
},
|
76 |
-
}
|
77 |
|
78 |
nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
|
79 |
|
@@ -150,66 +125,3 @@ def create_nlp_engine_with_transformers(
|
|
150 |
|
151 |
return nlp_engine, registry
|
152 |
|
153 |
-
|
154 |
-
def create_nlp_engine_with_flair(
|
155 |
-
model_path: str,
|
156 |
-
) -> Tuple[NlpEngine, RecognizerRegistry]:
|
157 |
-
"""
|
158 |
-
Instantiate an NlpEngine with a FlairRecognizer and a small spaCy model.
|
159 |
-
The FlairRecognizer would return results from Flair models, the spaCy model
|
160 |
-
would return NlpArtifacts such as POS and lemmas.
|
161 |
-
:param model_path: Flair model path.
|
162 |
-
"""
|
163 |
-
from flair_recognizer import FlairRecognizer
|
164 |
-
|
165 |
-
registry = RecognizerRegistry()
|
166 |
-
registry.load_predefined_recognizers()
|
167 |
-
|
168 |
-
# there is no official Flair NlpEngine, hence we load it as an additional recognizer
|
169 |
-
|
170 |
-
if not spacy.util.is_package("en_core_web_sm"):
|
171 |
-
spacy.cli.download("en_core_web_sm")
|
172 |
-
# Using a small spaCy model + a Flair NER model
|
173 |
-
flair_recognizer = FlairRecognizer(model_path=model_path)
|
174 |
-
nlp_configuration = {
|
175 |
-
"nlp_engine_name": "spacy",
|
176 |
-
"models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
|
177 |
-
}
|
178 |
-
registry.add_recognizer(flair_recognizer)
|
179 |
-
registry.remove_recognizer("SpacyRecognizer")
|
180 |
-
|
181 |
-
nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
|
182 |
-
|
183 |
-
return nlp_engine, registry
|
184 |
-
|
185 |
-
|
186 |
-
def create_nlp_engine_with_azure_ai_language(ta_key: str, ta_endpoint: str):
|
187 |
-
"""
|
188 |
-
Instantiate an NlpEngine with a TextAnalyticsWrapper and a small spaCy model.
|
189 |
-
The TextAnalyticsWrapper would return results from calling Azure Text Analytics PII, the spaCy model
|
190 |
-
would return NlpArtifacts such as POS and lemmas.
|
191 |
-
:param ta_key: Azure Text Analytics key.
|
192 |
-
:param ta_endpoint: Azure Text Analytics endpoint.
|
193 |
-
"""
|
194 |
-
from azure_ai_language_wrapper import AzureAIServiceWrapper
|
195 |
-
|
196 |
-
if not ta_key or not ta_endpoint:
|
197 |
-
raise RuntimeError("Please fill in the Text Analytics endpoint details")
|
198 |
-
|
199 |
-
registry = RecognizerRegistry()
|
200 |
-
registry.load_predefined_recognizers()
|
201 |
-
|
202 |
-
azure_ai_language_recognizer = AzureAIServiceWrapper(
|
203 |
-
ta_endpoint=ta_endpoint, ta_key=ta_key
|
204 |
-
)
|
205 |
-
nlp_configuration = {
|
206 |
-
"nlp_engine_name": "spacy",
|
207 |
-
"models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
|
208 |
-
}
|
209 |
-
|
210 |
-
nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
|
211 |
-
|
212 |
-
registry.add_recognizer(azure_ai_language_recognizer)
|
213 |
-
registry.remove_recognizer("SpacyRecognizer")
|
214 |
-
|
215 |
-
return nlp_engine, registry
|
|
|
48 |
return nlp_engine, registry
|
49 |
|
50 |
|
51 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
52 |
|
53 |
nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
|
54 |
|
|
|
125 |
|
126 |
return nlp_engine, registry
|
127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|