from keybert import KeyBERT import spacy import string from spacy import displacy from pathlib import Path from keyword_extraction import keyword_extract from keyphrase_extraction import get_top_key_phrases, display_key_phrases from word import show_gram_plot nlp = spacy.load("en_core_web_sm") doc = """ CIHI Database that collects, administrative, clinical and demographic information on hospital discharges (including deaths, sign-outs and transfers). Some provinces and territories also use the DAD to capture day surgery. Alberta The discharge abstract database is a database for information on all AHS separations for acute care institutions, including discharges, deaths, sign-outs and transfers. British Columbia Data on discharges, transfers and deaths of in-patients and day surgery patients from acute care hospitals in BC. All Canadian hospitals (except those in Quebec) submit their separations records directly to the Canadian Institute of Health information (CIHI) for inclusion in the Discharge Abstract Database (DAD). The database contains demographic, administrative and clinical data for hospital discharges (inpatient acute, chronic, rehabilitation) and day surgeries. A provincial data set, including various CIHI value-added elements (such as case mix groups, and resource intensity weights) is released on a monthly basis to the respective Ministries of Health. The DAD data files which Population Data BC receives include the CIHI variables. Population Data BC receives these data once per year. Manitoba Health data maintained by Manitoba Health consisting of hospital forms/computerized records containing summaries of demographic and clinical information (e.g., gender, postal code, diagnoses and procedure codes) completed at the point of discharge from the hospital. Several hundred thousand abstracts per year are submitted for all separations from acute and chronic care facilities in Manitoba and for all Manitobans admitted to out-of-province facilities. The Hospital Abstracts Data includes records of both Manitoba residents and non-Manitoba residents hospitalized in Manitoba facilities and information about inpatient and day surgery services. New Brunswick Patient discharge information from New Brunswick hospitals. Captures administrative, clinical and demographic information including discharges, deaths, sign-outs, and transfers. Newfoundland and Labrador The Provincial Discharge Abstract Database (PDAD) is the NLCHI dataset that contains demographic, clinical and administrative data collected at hospitals when patients are discharged from inpatient and surgical day care services and submitted to the CIHI Discharge Abstract Database. The PDAD captures information regarding hospitalizations of both residents of NL and non-residents receiving care in NL. Nova Scotia Contains information on each hospital admission recorded in a Nova Scotia hospital Ontario The Discharge Abstract Database is a database for information on all separation from acute care institutions within a fiscal year (April 1st to March 31st). Data is received directly from acute care facilities or from their respective health/regional authority or ministry/department of health. Saskatchewan Captures administrative, clinical and demographic information on discharges for acute care facilities (including deaths, sign-outs and transfers). """ doc = doc.translate(str.maketrans('', '', string.punctuation)) # run word count show_gram_plot(doc, 1, 10, save_output=f'results/DAD/{1}_gram.png') show_gram_plot(doc, 2, 10, save_output=f'results/DAD/{2}_gram.png') show_gram_plot(doc, 3, 10, save_output=f'results/DAD/{3}_gram.png') # run named entity recognition spacy_doc = nlp(doc) # Create a visualization of named entities svg = displacy.render(spacy_doc, style="ent", jupyter=False, page=True) filename = Path('results/DAD/ner.html') filename.open('w', encoding='utf-8').write(svg) # run keyword extraction kw_model = KeyBERT() keyword_extract(descriptions, kw_model, 1, save_output=f'results/DAD/{1}_keyword.png') keyword_extract(descriptions, kw_model, 2, save_output=f'results/DAD/{2}_keyword.png') keyword_extract(descriptions, kw_model, 3, save_output=f'results/DAD/{3}_keyword.png') keywords = kw_model.extract_keywords(doc, highlight=True) print(keywords) # run key phrase extraction get_top_key_phrases(doc, 10, save_output=f'results/DAD/top_keyphrase.png') display_key_phrases(doc, save_output='results/DAD/key_phrase.html')