File size: 9,906 Bytes
b1fa23d
466d5bb
 
b1fa23d
466d5bb
b1fa23d
47473dd
466d5bb
 
 
 
 
b1fa23d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cc281d5
b1fa23d
f48a49c
 
b1fa23d
47473dd
 
 
 
 
 
 
 
 
 
b1fa23d
e9b52b9
 
 
 
 
 
b1fa23d
 
 
 
1f1d19b
b1fa23d
 
 
 
 
c65f2f6
cc281d5
 
 
c65f2f6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cc281d5
 
c65f2f6
 
 
 
 
 
 
 
 
 
cc281d5
c65f2f6
 
 
 
 
b1fa23d
 
 
 
 
 
5aac296
 
 
 
 
 
 
 
 
b1fa23d
 
 
 
 
 
 
1f1d19b
b1fa23d
1f1d19b
47473dd
 
 
1f1d19b
47473dd
1f1d19b
 
 
 
 
 
 
 
 
 
 
 
 
5aac296
47473dd
 
b1fa23d
47473dd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b1fa23d
 
1f1d19b
47473dd
 
 
 
 
 
 
 
b1fa23d
 
9e76abd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b1fa23d
 
 
1f1d19b
e9b52b9
b1fa23d
e9b52b9
 
 
b1fa23d
e9b52b9
5909acd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
# !pip install mistune
import mistune
from mistune.plugins.table import table
from jinja2 import Template
import re
import os
from urllib.parse import urlparse

def md_to_html(md_text):
    renderer = mistune.HTMLRenderer()
    markdown_renderer = mistune.Markdown(renderer, plugins=[table])
    html_content = markdown_renderer(md_text)
    return html_content.replace('\n', '')

####------------------------------ OPTIONAL--> User id and persistant data storage-------------------------------------####
from datetime import datetime
import psycopg2

from dotenv import load_dotenv, find_dotenv

# Load environment variables from .env file
load_dotenv("keys.env")

TOGETHER_API_KEY = os.getenv('TOGETHER_API_KEY')
BRAVE_API_KEY = os.getenv('BRAVE_API_KEY')
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
HELICON_API_KEY = os.getenv("HELICON_API_KEY")
SUPABASE_USER = os.environ['SUPABASE_USER']
SUPABASE_PASSWORD = os.environ['SUPABASE_PASSWORD']

def insert_data(user_id, user_query, subtopic_query, response, html_report):
    # Connect to your database
    conn = psycopg2.connect(
    dbname="postgres",
    user=SUPABASE_USER,
    password=SUPABASE_PASSWORD,
    host="aws-0-us-west-1.pooler.supabase.com",
    port="5432"
)
    cur = conn.cursor()
    insert_query = """
    INSERT INTO research_pro_chat_v2 (user_id, user_query, subtopic_query, response, html_report, created_at)
    VALUES (%s, %s, %s, %s, %s, %s);
    """
    cur.execute(insert_query, (user_id,user_query, subtopic_query, response, html_report, datetime.now()))
    conn.commit()
    cur.close()
    conn.close()

####-----------------------------------------------------END----------------------------------------------------------####


import ast
from fpdf import FPDF
import re
import pandas as pd
import nltk
import requests
import json
from retry import retry
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from brave import Brave
from fuzzy_json import loads
from half_json.core import JSONFixer
from openai import OpenAI
from together import Together

llm_default_small = "meta-llama/Llama-3-8b-chat-hf"
llm_default_medium = "meta-llama/Llama-3-70b-chat-hf"

SysPromptData = """You are expert in information extraction from the given context.
                    Steps to follow:
                    1. Check if relevant factual data regarding <USER QUERY> is present in the <SCRAPED DATA>.
                       - IF YES, extract the maximum relevant factual information related to <USER QUERY> from the <SCRAPED DATA>.
                       - IF NO, then return "N/A"
                    
                    Rules to follow:
                    - Return N/A if information is not present in the scraped data.
                    - FORGET EVERYTHING YOU KNOW, Only output information that is present in the scraped data, DO NOT MAKE UP INFORMATION
                """
SysPromptDefault = "You are an expert AI, complete the given task. Do not add any additional comments."
SysPromptSearch = """You are a search query generator, create a concise Google search query, focusing only on the main topic and omitting additional redundant details, include year if necessory, 2024, Do not add any additional comments. OUTPUT ONLY THE SEARCH QUERY
                        #Additional instructions:
                        ##Use the following search operators if necessory
                        OR #to cover multiple topics
                        * #wildcard to match any word or phrase
                        AND #to include specific topics."""

import tiktoken # Used to limit tokens
encoding = tiktoken.encoding_for_model("gpt-3.5-turbo") # Instead of Llama3 using available option/ replace if found anything better

def limit_tokens(input_string, token_limit=7500):
    """
    Limit tokens sent to the model
    """
    return encoding.decode(encoding.encode(input_string)[:token_limit])

together_client = OpenAI(
        api_key=TOGETHER_API_KEY, 
        base_url="https://together.hconeai.com/v1", 
        default_headers={ "Helicone-Auth": f"Bearer {HELICON_API_KEY}"})

groq_client = OpenAI(
        api_key=GROQ_API_KEY, 
        base_url="https://groq.hconeai.com/openai/v1", 
        default_headers={ "Helicone-Auth": f"Bearer {HELICON_API_KEY}"})

# Groq model names
llm_default_small = "llama3-8b-8192"
llm_default_medium = "llama3-70b-8192"

# Together Model names (fallback)
llm_fallback_small = "meta-llama/Llama-3-8b-chat-hf"
llm_fallback_medium = "meta-llama/Llama-3-70b-chat-hf"

### ------END OF LLM CONFIG-------- ###

def together_response(message, model = llm_default_small, SysPrompt = SysPromptDefault, temperature=0.2, frequency_penalty =0.1, max_tokens= 2000):
    
    messages=[{"role": "system", "content": SysPrompt},{"role": "user", "content": message}]
    params = {
      "model": model,
      "messages": messages,
      "temperature": temperature,
      "frequency_penalty": frequency_penalty,
      "max_tokens": max_tokens
    }
    try:
      response = groq_client.chat.completions.create(**params)
      return response.choices[0].message.content
    
    except Exception as e:
      print(f"Error calling GROQ API: {e}")
      params["model"] = llm_fallback_small if model == llm_default_small else llm_fallback_medium 
      response = together_client.chat.completions.create(**params)
      return response.choices[0].message.content

def json_from_text(text):
    """
    Extracts JSON from text using regex and fuzzy JSON loading.
    """
    try:
      return json.loads(text)
    except:
      match = re.search(r'\{[\s\S]*\}', text)
      if match:
        json_out = match.group(0)
      else:
        json_out = text
      # Use Fuzzy JSON loading
      return loads(json_out)

def remove_stopwords(text):
    stop_words = set(stopwords.words('english'))
    words = word_tokenize(text)
    filtered_text = [word for word in words if word.lower() not in stop_words]
    return ' '.join(filtered_text)

def rephrase_content(data_format, content, query):

    if data_format == "Structured data":
        return together_response(f"""
            <SCRAPED DATA>{content}</SCRAPED DATA>
            extract the maximum relevant factual information covering all aspects of <USER QUERY>{query}</USER QUERY> ONLY IF AVAILABLE in the scraped data.""",
            SysPrompt=SysPromptData,
            max_tokens=900,
        )
    elif data_format == "Quantitative data":
        return together_response(
            f"return only the numerical or quantitative data regarding the query: {{{query}}} structured into .md tables, using the scraped context:{{{limit_tokens(content,token_limit=1000)}}}",
            SysPrompt=SysPromptData,
            max_tokens=500,
        )
    else:
        return together_response(
            f"return only the factual information regarding the query: {{{query}}} using the scraped context:{{{limit_tokens(content,token_limit=1000)}}}",
            SysPrompt=SysPromptData,
            max_tokens=500,
        )
        
def extract_main_content(url):
    if url:
        try:
            result = urlparse(url)
            if all([result.scheme, result.netloc]):
                # Prepare query parameters
                params = {
                    "url": url,
                    "favor_precision": False,
                    "favor_recall": False,
                    "output_format": "markdown",
                    "target_language": "en",
                    "include_tables": True,
                    "include_images": False,
                    "include_links": False,
                    "deduplicate": True,
                }

                # Make request to FastAPI endpoint
                response = requests.get("https://pvanand-web-scraping.hf.space/extract-article", params=params)

                if response.status_code == 200:
                    return response.json()["article"]
                else:
                    return ""
        except:
            return ""
    return ""

def process_content(data_format, url, query):
    content = extract_main_content(url)
    if content:
        rephrased_content = rephrase_content(
            data_format=data_format,
            content=limit_tokens(content, token_limit=4000),
            query=query,
        )
        return rephrased_content, url
    return "", url

def fetch_and_extract_content(
    data_format: str, query: str, urls: List[str], num_refrences: int = 8
) -> List[Tuple[str | None, str]]:
    """
    Asynchronously makeing request to urls and doing further process
    """
    all_text_with_urls = []
    start_url = 0
    while (len(all_text_with_urls) != num_refrences) and (start_url < len(urls)):
        end_url = start_url + (num_refrences - len(all_text_with_urls))
        urls_subset = urls[start_url:end_url]
        with ThreadPoolExecutor(max_workers=len(urls_subset)) as executor:
            future_to_url = {
                executor.submit(process_content, data_format, url, query): url
                for url in urls_subset
            }
            all_text_with_urls += [
                future.result()
                for future in as_completed(future_to_url)
                if future.result()[0] != ""
            ]
        start_url = end_url

    return all_text_with_urls


@retry(tries=3, delay=0.25)
def search_brave(query, num_results=5):
    cleaned_query = re.sub(r'[^a-zA-Z0-9]+', '', query)
    search_query = together_response(cleaned_query, model=llm_default_small, SysPrompt=SysPromptSearch, max_tokens = 25).strip()
    cleaned_search_query = re.sub(r'[^a-zA-Z0-9*]+', '', search_query)
    brave = Brave(BRAVE_API_KEY)
    search_results = brave.search(q=cleaned_search_query, count=num_results)
    return [url.__str__() for url in search_results.urls],cleaned_search_query