from textblob import TextBlob import spacy from spacy import displacy import pandas as pd import seaborn as sns import textrank import matplotlib.pyplot as plt from pathlib import Path from nltk.tokenize import sent_tokenize, word_tokenize from nltk.corpus import stopwords from nltk.probability import FreqDist from nltk.tokenize import word_tokenize from nltk.stem import PorterStemmer from sklearn.metrics.pairwise import cosine_similarity import networkx as nx import matplotlib.pyplot as plt import numpy as np import pytextrank # Load the pre-trained NLP model nlp = spacy.load("en_core_web_sm") nlp.add_pipe('textrank') # Sample text to analyze text = """ Database that collects, administrative, clinical and demographic information on hospital discharges (including deaths, sign-outs and transfers). Some provinces and territories also use the DAD to capture day surgery. The discharge abstract database is a database for information on all AHS separations for acute care institutions, including discharges, deaths, sign-outs and transfers. Data on discharges, transfers and deaths of in-patients and day surgery patients from acute care hospitals in BC. All Canadian hospitals (except those in Quebec) submit their separations records directly to the Canadian Institute of Health information (CIHI) for inclusion in the Discharge Abstract Database (DAD). The database contains demographic, administrative and clinical data for hospital discharges (inpatient acute, chronic, rehabilitation) and day surgeries. A provincial data set, including various CIHI value-added elements (such as case mix groups, and resource intensity weights) is released on a monthly basis to the respective Ministries of Health. The DAD data files which Population Data BC receives include the CIHI variables. Population Data BC receives these data once per year. Health data maintained by Manitoba Health consisting of hospital forms/computerized records containing summaries of demographic and clinical information (e.g., gender, postal code, diagnoses and procedure codes) completed at the point of discharge from the hospital. Several hundred thousand abstracts per year are submitted for all separations from acute and chronic care facilities in Manitoba and for all Manitobans admitted to out-of-province facilities. The Hospital Abstracts Data includes records of both Manitoba residents and non-Manitoba residents hospitalized in Manitoba facilities and information about inpatient and day surgery services. Patient discharge information from New Brunswick hospitals. Captures administrative, clinical and demographic information including discharges, deaths, sign-outs, and transfers. The Provincial Discharge Abstract Database (PDAD) is the NLCHI dataset that contains demographic, clinical and administrative data collected at hospitals when patients are discharged from inpatient and surgical day care services and submitted to the CIHI Discharge Abstract Database. The PDAD captures information regarding hospitalizations of both residents of NL and non-residents receiving care in NL. Contains information on each hospital admission recorded in a Nova Scotia hospital The Discharge Abstract Database is a database for information on all separation from acute care institutions within a fiscal year (April 1st to March 31st). Data is received directly from acute care facilities or from their respective health/regional authority or ministry/department of health. Captures administrative, clinical and demographic information on discharges for acute care facilities (including deaths, sign-outs and transfers). """ def get_top_key_phrases(text, top_n, save_output): # Process the text doc = nlp(text) # show the score of key phrases # phrases_ranking = {phrase.text: phrase.rank for phrase in doc._.phrases} phrases = list(zip(*phrases_ranking.items()))[0] scores = list(zip(*phrases_ranking.items()))[1] keyword_df = pd.DataFrame({'words': phrases[:top_n], 'scores': scores[:top_n]}) plt.title("Word Count") plt.figure(figsize=(24, 8)) plt.yticks(fontsize=15) sns.barplot(data=keyword_df, y='words', x='scores', palette='blend:#7AB,#EDA') plt.xlabel("scores", fontsize=15) plt.ylabel("words", fontsize=15) plt.savefig(save_output, dpi=300, bbox_inches="tight") plt.close() def visualize_textrank(text): # Get text # Generate TextRank tr = textrank.TextRank() tr.calculate_scores(text) # Get top 10 words words = [w for w, s in tr.top_words(10)] # Create graph G = nx.Graph() # Add nodes for w in words: G.add_node(w) # Find co-occurrence counts counts = {} for i in range(len(words) - 1): w1 = words[i] w2 = words[i + 1] key = (w1, w2) counts[key] = counts.get(key, 0) + 1 # Add edges with weights for key, count in counts.items(): w1, w2 = key G.add_edge(w1, w2, weight=count) # Draw graph with weighted edges nx.draw(G, with_labels=True, width=list(e[2]['weight'] for e in G.edges())) plt.show() def display_key_phrases(text, save_output): text = text.replace('\n', ' \n') doc = nlp(text) key_phrases = [{'start': chunk.start_char, 'end': chunk.end_char, 'label': str(round(phrase.rank, 2))} for phrase in doc._.phrases for chunk in phrase.chunks] # generate displacy html # max_rank = float(key_phrases[0]['label']) min_rank = float(key_phrases[-1]['label']) step = 50/max_rank colors = {str(phrase['label']): f'hsl(121, 100%, {100-float(phrase["label"])*step}%)' for phrase in key_phrases} options = {'ents': [color for color in colors.keys()], 'colors': colors} # Create a list of spans to highlight sentence = [ {'text': text, 'ents': key_phrases, 'title': None} ] # Create a visualization of the text with highlighted key phrases svg = displacy.render(sentence, style="ent", options=options, manual=True, page=True) filename = Path(save_output) filename.open('w', encoding='utf-8').write(svg) return svg if __name__ == '__main__': visualize_textrank(text) # get_top_key_phrases(text, 10, 'test_results/keyphrase.png') # display_key_phrases(text)