Update helper_functions_api.py

helper_functions_api.py

@@ -4,7 +4,7 @@ from mistune.plugins.table import table
 from jinja2 import Template
 import re
 import os
-import hrequests
+import requests
 
 def md_to_html(md_text):
     renderer = mistune.HTMLRenderer()
@@ -183,16 +183,20 @@ def rephrase_content(data_format, content, query):
             max_tokens=500,
         )
         
+class Scraper:
+    def __init__(self, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"):
+        self.session = requests.Session()
+        self.session.headers.update({"User-Agent": user_agent})
 
-@retry(tries=3, delay=1)
-def fetch_content(url):
-    try:
-        response = hrequests.get(url)
-        if response.status_code == 200:
-            return response.text
-    except Exception as e:
-        print(f"Error fetching page content for {url}: {e}")
-    return None
+    @retry(tries=3, delay=1)
+    def fetch_content(self, url):
+        try:
+            response = self.session.get(url, timeout=2)
+            if response.status_code == 200:
+                return response.text
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching page content for {url}: {e}")
+        return None
 
 def extract_main_content(html):
     extracted = trafilatura.extract(
@@ -211,7 +215,8 @@ def extract_main_content(html):
         return ""
 
 def process_content(data_format, url, query):
-    html_content = fetch_content(url)
+    scraper = Scraper()
+    html_content = scraper.fetch_content(url)
     if html_content:
         content = extract_main_content(html_content)
         if content:
@@ -263,4 +268,4 @@ def search_brave(query, num_results=5):
 #     cleaned_search_query = re.sub(r'[^\w\s]', '', search_query).strip() #re.sub(r'[^a-zA-Z0-9*]+', '', search_query)
 #     brave = Brave(BRAVE_API_KEY)
 #     search_results = brave.search(q=cleaned_search_query, count=num_results)
-#     return [url.__str__() for url in search_results.urls],cleaned_search_query
+#     return [url.__str__() for url in search_results.urls],cleaned_search_query