Update helper_functions_api.py

helper_functions_api.py

@@ -4,7 +4,7 @@ from mistune.plugins.table import table
 from jinja2 import Template
 import re
 import os
-import requests
+import hrequests
 
 def md_to_html(md_text):
     renderer = mistune.HTMLRenderer()
@@ -183,20 +183,15 @@ def rephrase_content(data_format, content, query):
             max_tokens=500,
         )
         
-class Scraper:
-    def __init__(self, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"):
-        self.session = requests.Session()
-        self.session.headers.update({"User-Agent": user_agent})
 
-    @retry(tries=3, delay=1)
-    def fetch_content(self, url):
-        try:
-            response = self.session.get(url, timeout=2)
-            if response.status_code == 200:
-                return response.text
-        except requests.exceptions.RequestException as e:
-            print(f"Error fetching page content for {url}: {e}")
-        return None
+def fetch_content(url):
+    try:
+        response = hrequests.get(url)
+        if response.status_code == 200:
+            return response.text
+    except Exception as e:
+        print(f"Error fetching page content for {url}: {e}")
+    return None
 
 def extract_main_content(html):
     extracted = trafilatura.extract(
@@ -215,8 +210,7 @@ def extract_main_content(html):
         return ""
 
 def process_content(data_format, url, query):
-    scraper = Scraper()
-    html_content = scraper.fetch_content(url)
+    html_content = fetch_content(url)
     if html_content:
         content = extract_main_content(html_content)
         if content:
@@ -268,4 +262,4 @@ def search_brave(query, num_results=5):
 #     cleaned_search_query = re.sub(r'[^\w\s]', '', search_query).strip() #re.sub(r'[^a-zA-Z0-9*]+', '', search_query)
 #     brave = Brave(BRAVE_API_KEY)
 #     search_results = brave.search(q=cleaned_search_query, count=num_results)
-#     return [url.__str__() for url in search_results.urls],cleaned_search_query
+#     return [url.__str__() for url in search_results.urls],cleaned_search_query