Upload three files for url RAG

app.py

@@ -0,0 +1,67 @@
+import streamlit as st
+import utils
+
+
+# https://github.com/serkanyasr/RAG-with-LangChain-URL-PDF/blob/main/utils.py
+
+
+st.set_page_config(layout="wide")
+st.markdown("<h1 style='font-size:24px;'>RAG with LangChain & GenAI: Any url</h1>", unsafe_allow_html=True)
+# st.title("RAG with LangChain & GenAI: Any url")
+
+# URL text box for user input
+url_input = st.text_input("Enter a URL to be queried:", "")
+
+# Input text box for user input
+user_input = st.text_input("Enter your Question below:", "")
+
+# Display the user input
+# st.write("You entered:", user_input)
+# st.write("URL entered:", url_input)
+sumbit_btn = st.button(label="Submit",key="url_btn")
+
+if sumbit_btn:
+    with st.spinner("Processing..."):
+        st.success("Response: Answering with RAG...")
+        response = utils.rag_with_url(url_input,user_input)
+        st.markdown(response)
+
+
+
+
+
+
+            
+
+# st.title("Retrieval-Augmented Generation (RAG) with LangChain : PDF ")
+# st.divider()
+
+# col_input , col_rag , col_normal = st.columns([3,5,5])
+# with col_input:
+#     selected_file = st.file_uploader("PDF File", type=["pdf"])
+#     st.divider()
+#     prompt = st.text_input("Prompt",key="pdf_prompt")
+#     st.divider()
+#     sumbit_btn = st.button(label="Submit",key="pdf_btn")
+
+# if sumbit_btn:
+#     with col_rag:
+#         with st.spinner("Processing..."):
+#             st.success("Response: Answering with RAG...")
+#             response,relevant_documents = utils.rag_with_pdf(file_path=f"./data/{selected_file.name}",
+#                                                                   prompt=prompt)
+#             st.markdown(response)
+#             st.divider()
+#             st.info("Documents")
+#             for doc in relevant_documents:
+#                 st.caption(doc.page_content)
+#                 st.markdown(f"Source: {doc.metadata}")
+#                 st.divider()
+
+#             with col_normal:
+#                 with st.spinner("Processing..."):
+#                     st.info("Response: Answering without RAG...")
+#                     response = utils.ask_gemini(prompt)
+#                     st.markdown(response)
+#                     st.divider()
+            

requirements.txt

@@ -0,0 +1,20 @@
+langchain
+python-dotenv
+langchain-openai
+langchain-cohere
+langchain-google-genai
+openai
+streamlit
+python-dotenv
+bs4
+cohere
+faiss-cpu
+pypdf
+huggingface_hub
+langchain_community
+
+unstructured
+tiktoken
+libmagic
+python-magic
+python-magic-bin

utils.py

@@ -0,0 +1,63 @@
+from langchain_google_genai import ChatGoogleGenerativeAI 
+from langchain_openai import OpenAIEmbeddings
+from langchain_cohere import CohereEmbeddings
+from langchain_openai import OpenAI
+from langchain_community.document_loaders.web_base import WebBaseLoader
+from langchain_community.document_loaders.pdf import PyPDFLoader
+from langchain_community.vectorstores.faiss import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings.huggingface import HuggingFaceInferenceAPIEmbeddings
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+GEMINI_API_KEY = os.getenv("GOOGLE_AI_API_KEY")
+HF_API_KEY = os.getenv("HF_API_KEY")
+
+llm_gemini = ChatGoogleGenerativeAI( google_api_key= GEMINI_API_KEY, model="gemini-pro")
+embeddings_hf = HuggingFaceInferenceAPIEmbeddings(api_key=HF_API_KEY, model="sentence-transformers/all-MiniLM-16-v2")
+
+# OPEN_AI_API_KEY = os.getenv("OPEN_AI_API_KEY")
+# COHERE_API_KEY = os.getenv("COHERE_API_KEY")
+# llm_openai = OpenAI(api_key=OPEN_AI_API_KEY, model="gpt-3.5-turbo")
+# embeddings_open_ai = OpenAIEmbeddings(api_key=OPEN_AI_API_KEY) # OPEN_AI
+# embeddings_cohere = CohereEmbeddings(api_key=COHERE_API_KEY,model="embed-multilingual-v3.0") # embed-english-v3.0
+
+
+
+def ask_gemini(prompt):
+    AI_Respose = llm_gemini.invoke(prompt)
+    return AI_Respose.content
+
+
+
+def rag_with_url(target_url, prompt):
+
+    loader = WebBaseLoader(target_url)
+    raw_document = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200, length_function=len)
+    splited_document = text_splitter.split_documents(raw_document)
+    vector_store = FAISS.from_documents(splited_document, embeddings_hf)
+    retriever = vector_store.as_retriever()
+    relevant_documents = retriever.get_relevant_documents(prompt)
+    final_prompt = prompt + " " + " ".join([doc.page_content for doc in relevant_documents])
+    AI_Respose = llm_gemini.invoke(final_prompt)
+
+    return AI_Respose.content
+
+
+
+# def rag_with_pdf(file_path, prompt):
+#     loader = PyPDFLoader(file_path)    
+#     raw_document = loader.load()
+#     text_splitter = RecursiveCharacterTextSplitter(chunk_size = 1000, chunk_overlap = 200, length_function = len)
+#     splited_document = text_splitter.split_documents(raw_document)
+#     vector_store = FAISS.from_documents(splited_document, embeddings_hf)
+#     retriever = vector_store.as_retriever()
+#     relevant_documents = retriever.get_relevant_documents(prompt)
+#     final_prompt = prompt + " " + " ".join([doc.page_content for doc in relevant_documents])
+#     AI_Respose = llm_gemini.invoke(final_prompt)
+#     return AI_Respose.content, relevant_documents
+