Update app.py

app.py

@@ -6,6 +6,7 @@ import google.generativeai as genai
 from langchain.prompts import PromptTemplate
 from langchain import LLMChain
 from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_community.document_loaders import WebBaseLoader
 
 os.environ["GOOGLE_API_KEY"] = os.getenv("GOOGLE_API_KEY")
 genai.configure(api_key=os.environ["GOOGLE_API_KEY"])
@@ -73,14 +74,38 @@ def get_pdf_text(pdf_docs):
             text += page.extract_text()
     return text
 
+def get_url_text(url_link):
+    try:
+        loader = WebBaseLoader(url_link)
+        loader.requests_per_second = 1
+        docs = loader.aload()
+        extracted_text = ""
+        for page in docs:
+            extracted_text += page.page_content
+        return extracted_text
+    except Exception as e:
+        print(f"Error fetching or processing URL: {e}")
+        return ""
+
 with st.sidebar:
     st.title("Add a file for CRETA memory:")
-    uploaded_file = st.file_uploader("Upload your PDF Files and Click on the Submit & Process Button", accept_multiple_files=True)
-    if st.button("Submit & Process"):
-        with st.spinner("Processing..."):
-            st.session_state["docs"] += get_pdf_text(uploaded_file)
-            st.success("Done")
+    uploaded_files = st.file_uploader("Upload your PDF Files and Click on the Submit & Process Button", accept_multiple_files=True)
+    uploaded_url = st.text_input("Paste the Documentation URL:")
     
+    if st.button("Submit & Process"):
+        if uploaded_files or uploaded_url:
+            with st.spinner("Processing..."):
+                if uploaded_files:
+                    st.session_state["docs"] += get_pdf_text(uploaded_files)
+                
+                if uploaded_url:
+                    st.session_state["docs"] += get_url_text(uploaded_url)
+                
+                st.success("Processing complete!")
+        else:
+            st.error("Please upload at least one PDF file or provide a URL.")
+            
+            
 # Create containers for chat history and user input
 response_container = st.container()
 container = st.container()
@@ -101,4 +126,5 @@ if st.session_state['generated']:
         for i in range(len(st.session_state['generated'])):
             if i != 0:
                 message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="adventurer")
-            message(st.session_state["generated"][i], key=str(i), avatar_style="bottts")
+            message(st.session_state["generated"][i], key=str(i), avatar_style="bottts")
+