deploy 1

Dockerfile.api

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get -y install git python3.10 python3-pip
+
+COPY requirements.txt .
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+WORKDIR /hugging-face-qa-bot
+COPY config/api/ config/api/
+COPY api/ api/
+
+EXPOSE 8000
+
+ENTRYPOINT [ "python3", "-m", "api" ]

Dockerfile.bot

@@ -0,0 +1,17 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get -y install git python3.10 python3-pip
+
+COPY requirements.txt .
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+WORKDIR /hugging-face-qa-bot
+COPY config/bot/ config/bot/
+COPY bot/ bot/
+
+ENTRYPOINT [ "python3", "-m", "bot" ]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,80 @@
----
-title: Bot
-emoji: 🏢
-colorFrom: indigo
-colorTo: indigo
-sdk: gradio
-sdk_version: 3.36.1
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Hugging Face Question Answering Bot
+
+This repository focuses on the development of a Hugging Face question answering bot that assists users in creating their own ML solutions and troubleshooting technical issues related to Hugging Face libraries. Our solution combines an efficient context retrieval mechanism powered by FAISS with Stanford's Alpaca 7B language model to provide accurate and contextually relevant guidance derived from the Hugging Face documentation. The bot is designed to operate entirely locally on a consumer device, ensuring both accessibility and privacy. 
+
+# Purpose
+The Hugging Face Question Answering Bot is designed to help users quickly find solutions to common problems and questions related to Hugging Face libraries. Whether you're just getting started with ML or you're an experienced developer looking for advanced guidance, the bot can help you get the information you need to succeed.
+
+# Example
+![Example](./assets/example.png)
+
+# Table of Contents
+- [Setting up the bot](#setting-up-the-bot)
+    - [Running in a Docker](#running-in-a-docker)
+    - [Running in a Python](#running-in-a-python)
+- [Development instructions](#development-instructions)
+- [Datasets](#dataset-list)
+
+## Setting up the bot
+First, you need to provide the necessary environmental variables and API keys in the .env file. 
+- `HUGGINGFACEHUB_API_TOKEN` - API key for HF Hub
+- `DISCORD_TOKEN` - API key for the bot application
+- `QUESTION_ANSWERING_MODEL_ID` - an ID of a model to be queried from HF Hub (in case of inference through API)
+- `EMBEDDING_MODEL_ID` - an ID of embedding model, used to create and query index on the documents
+- `INDEX_NAME` - directory where the index files are present after creation
+- `USE_DOCS_IN_CONTEXT` - allow context extration from documents
+- `ADD_SOURCES_TO_RESPONSE` - show references to documents that were used as a context for a given query
+- `USE_MESSEGES_IN_CONTEXT` - allow to use chat history for conversational experience
+- `NUM_LAST_MESSAGES` - number of messages used for the previous feature
+- `USE_NAMES_IN_CONTEXT` - use names of users in context
+- `ENABLE_COMMANDS` - allow command, e.g. channel cleanup
+- `DEBUG` - provides additional logging
+
+If you decide that you want to **run everthing locally** our current MVP recommends using Instructor large and Alpaca 7B with 4-bit quatization models. For this to properly work, you need to put the weights of the model in the `/bot/question_answering/` and set the `QUESTION_ANSWERING_MODEL_ID` variable to the name of the file that you just put in the aforementioned folder. Now, you should be able to run your own, local instance of the bot.
+
+### Running in a Docker
+```bash
+docker build -t <container-name> .
+docker run <container-name>
+# or simply:
+./run_docker.sh
+```
+
+### Running in a Python
+```bash
+pip install -r requirements.txt
+python3 -m bot
+```
+
+## Development Instructions
+
+We use `Python 3.10`
+
+To install all necessary Python packages, run the following command:
+
+```bash
+pip install -r requirements.txt
+```
+We use the pipreqsnb to generate the requirements.txt file. To install pipreqsnb, run the following command:
+
+```bash
+pip install pipreqsnb
+```
+To generate the requirements.txt file, run the following command:
+
+```bash
+pipreqsnb --force .
+```
+
+To run unit tests, you can use the following command:
+
+```bash
+pytest -o "testpaths=tests" --noconftest
+``` 
+
+## Dataset List
+
+Below is a list of the datasets used during development:
+- [Stack Overflow - Python](https://huggingface.co/datasets/KonradSzafer/stackoverflow_python_preprocessed)
+- [Stack Overflow - Linux](https://huggingface.co/datasets/KonradSzafer/stackoverflow_linux)
+- [Hugging Face Documentation](https://huggingface.co/docs)

api/__init__.py

@@ -0,0 +1,6 @@
+from dotenv import load_dotenv
+from api.logger import setup_logger
+
+
+setup_logger()
+load_dotenv(dotenv_path='config/api/.env')

api/__main__.py

@@ -0,0 +1,40 @@
+import uvicorn
+from fastapi import FastAPI
+
+from api.config import Config
+from api.question_answering import QAModel
+from api.logger import logger
+
+
+config = Config()
+app = FastAPI()
+qa_model = QAModel(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    use_docs_for_context=config.use_docs_for_context,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    num_relevant_docs=config.num_relevant_docs,
+    debug=config.debug
+)
+
+
+@app.get("/")
+def get_answer(question: str, messgages_context: str):
+    logger.info(
+        f"Received request with question: {question}" \
+        f"and context: {messgages_context}"
+    )
+    response = qa_model.get_answer(
+        question=question,
+        messages_context=messgages_context
+    )
+    return {
+        "answer": response.get_answer(),
+        "sources": response.get_sources_as_text()
+    }
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

api/config.py

@@ -0,0 +1,48 @@
+import os
+from dataclasses import dataclass, asdict
+from typing import Dict, Union
+from api.logger import logger
+
+
+def get_env(env_name: str, default = None) -> str:
+    env = os.getenv(env_name)
+    if not env:
+        if default:
+            logger.warning(
+                f'Environment variable {env_name} not found.' \
+                f'Using the default value: {default}.'
+            )
+            return default
+        else:
+            raise ValueError(f'Cannot parse: {env_name}')
+    else:
+        return env
+
+
+@dataclass
+class Config:
+    huggingface_token: str = get_env('HUGGINGFACEHUB_API_TOKEN')
+    question_answering_model_id: str = get_env('QUESTION_ANSWERING_MODEL_ID')
+    embedding_model_id: str = get_env('EMBEDDING_MODEL_ID')
+    index_repo_id: str = get_env('INDEX_REPO_ID')
+    use_docs_for_context: bool = eval(get_env('USE_DOCS_FOR_CONTEXT', 'True'))
+    add_sources_to_response: bool = eval(get_env('ADD_SOURCES_TO_RESPONSE', 'True'))
+    use_messages_in_context: bool = eval(get_env('USE_MESSAGES_IN_CONTEXT', 'True'))
+    num_relevant_docs: bool = eval(get_env('NUM_RELEVANT_DOCS', 3))
+    debug: bool = eval(get_env('DEBUG', 'True'))
+
+    def __post_init__(self):
+        # validate config
+        if not self.use_docs_for_context and self.add_sources_to_response:
+            raise ValueError('Cannot add sources to response if not using docs in context')
+        if self.num_relevant_docs < 1:
+            raise ValueError('num_relevant_docs must be greater than 0')
+        self.log()
+
+    def asdict(self) -> Dict:
+        return asdict(self)
+
+    def log(self) -> None:
+        logger.info('Config:')
+        for key, value in self.asdict().items():
+            logger.info(f'{key}: {value}')

api/logger.py

@@ -0,0 +1,14 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+def setup_logger() -> None:
+    """
+    Set up the logger.
+    """
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

api/question_answering/__init__.py

@@ -0,0 +1 @@
+from .qa_model import QAModel

api/question_answering/mocks.py

@@ -0,0 +1,39 @@
+from typing import Mapping, Optional, List, Any
+import os
+from langchain.llms.base import LLM
+
+class MockLocalBinaryModel(LLM):
+    """
+    Mock Local Binary Model class, used for generating the string "a".
+
+    Args:
+        model_id (str): The ID of the model to be mocked.
+
+    Attributes:
+        model_path (str): The path to the model to be mocked.
+        llm (str): The string "a".
+
+    Raises:
+        ValueError: If the model_path does not exist.
+    """
+
+    model_path: str = None
+    llm: str = "READY TO MOCK"
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        self.model_path = f'bot/question_answering/{model_id}'
+        if not os.path.exists(self.model_path):
+            raise ValueError(f'{self.model_path} does not exist')
+
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        return self.llm
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"name_of_model": self.model_path}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_path

api/question_answering/qa_model.py

@@ -0,0 +1,267 @@
+import os
+import json
+import requests
+import subprocess
+import torch
+import transformers
+from urllib.parse import quote
+from typing import Mapping, Optional, List, Any
+from huggingface_hub import snapshot_download
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from langchain import PromptTemplate, HuggingFaceHub, LLMChain
+from langchain.llms import HuggingFacePipeline
+from langchain.llms.base import LLM
+from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceHubEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+from llama_cpp import Llama
+
+from api.logger import logger
+from api.question_answering.response import Response
+
+
+class LocalBinaryModel(LLM):
+    model_id: str = None
+    llm: Llama = None
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        model_path = f'api/question_answering/{model_id}'
+        if not os.path.exists(model_path):
+            raise ValueError(f'{model_path} does not exist')
+        self.model_id = model_id
+        self.llm = Llama(model_path=model_path, n_ctx=4096)
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        prompt = f'Q: {prompt} A: '
+        output = self.llm(
+            prompt,
+            max_tokens=1024,
+            stop=['Q:'],
+            echo=False
+        )
+        output_text = output['choices'][0]['text']
+        return output_text
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"name_of_model": self.model_id}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_id
+
+
+class TransformersPipelineModel(LLM):
+    model_id: str = None
+    pipeline: str = None
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        self.model_id = model_id
+
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id,
+            torch_dtype=torch.bfloat16,
+            trust_remote_code=True,
+            load_in_8bit=False,
+            device_map="auto",
+            resume_download=True,
+        )
+        self.pipeline = transformers.pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            max_new_tokens=2048,
+        )
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        output_text = self.pipeline(prompt)[0]['generated_text']
+        output_text = output_text.replace(prompt+'\n', '')
+        return output_text
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"name_of_model": self.model_id}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_id
+
+
+class APIServedModel(LLM):
+    model_url: str = None
+    debug: bool = None
+
+    def __init__(self, model_url: str = None, debug: bool = None):
+        super().__init__()
+        if model_url[-1] == '/':
+            raise ValueError('URL should not end with a slash - "/"')
+        self.model_url = model_url
+        self.debug = debug
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        prompt_encoded = quote(prompt, safe='')
+        url = f'{self.model_url}/?prompt={prompt_encoded}'
+        if self.debug:
+            logger.info(f'URL: {url}')
+        try:
+            response = requests.get(url, timeout=1200, verify=False)
+            response.raise_for_status() 
+            output_text = json.loads(response.content)['output_text']
+            return output_text
+        except Exception as err:
+            logger.error(f'Error: {err}')
+            return f'Error: {err}'
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"name_of_model": f'model url: {self.model_url}'}
+
+    @property
+    def _llm_type(self) -> str:
+        return 'api_model'
+
+
+
+class QAModel():
+    """
+    QAModel class, used for generating answers to questions.
+
+    Args:
+        llm_model_id (str): The ID of the LLM model to be used.
+        embedding_model_id (str): The ID of the embedding model to be used.
+        index_repo_id (str): The ID of the index repository to be used.
+        run_locally (bool, optional): Whether to run the models locally or on the Hugging Face hub. Defaults to True.
+        use_docs_for_context (bool, optional): Whether to use relevant documents as context for generating answers.
+        Defaults to True.
+        use_messages_for_context (bool, optional): Whether to use previous messages as context for generating answers.
+        Defaults to True.
+        debug (bool, optional): Whether to log debug information. Defaults to False.
+
+    Attributes:
+        use_docs_for_context (bool): Whether to use relevant documents as context for generating answers.
+        use_messages_for_context (bool): Whether to use previous messages as context for generating answers.
+        debug (bool): Whether to log debug information.
+        llm_model (Union[LocalBinaryModel, HuggingFacePipeline, HuggingFaceHub]): The LLM model to be used.
+        embedding_model (Union[HuggingFaceInstructEmbeddings, HuggingFaceHubEmbeddings]): The embedding model to be used.
+        prompt_template (PromptTemplate): The prompt template to be used.
+        llm_chain (LLMChain): The LLM chain to be used.
+        knowledge_index (FAISS): The FAISS index to be used.
+
+    """
+    def __init__(
+        self,
+        llm_model_id: str,
+        embedding_model_id: str,
+        index_repo_id: str,
+        use_docs_for_context: bool = True,
+        add_sources_to_response: bool = True,
+        use_messages_for_context: bool = True,
+        num_relevant_docs: int = 3,
+        debug: bool = False
+    ):
+        super().__init__()
+        self.use_docs_for_context = use_docs_for_context
+        self.add_sources_to_response = add_sources_to_response
+        self.use_messages_for_context = use_messages_for_context
+        self.num_relevant_docs = num_relevant_docs
+        self.debug = debug
+
+        if 'local_models/' in llm_model_id:
+            logger.info('using local binary model')
+            self.llm_model = LocalBinaryModel(
+                model_id=llm_model_id
+            )
+        elif 'api_models/' in llm_model_id:
+            logger.info('using api served model')
+            self.llm_model = APIServedModel(
+                model_url=llm_model_id.replace('api_models/', ''),
+                debug=self.debug
+            )
+        else:
+            logger.info('using transformers pipeline model')
+            self.llm_model = TransformersPipelineModel(
+                model_id=llm_model_id
+            )
+
+        prompt_template = \
+            "### Instruction:\n" \
+            "Give an answer that contains all the necessary information for the question.\n" \
+            "If the context contains necessary information to answer question, use it to generate an appropriate response.\n" \
+            "{context}\n### Input:\n{question}\n### Response:"
+
+        prompt = PromptTemplate(
+            template=prompt_template,
+            input_variables=['question', 'context']
+        )
+        self.llm_chain = LLMChain(prompt=prompt, llm=self.llm_model)
+
+        if self.use_docs_for_context:
+            logger.info(f'Downloading {index_repo_id}')
+            snapshot_download(
+                repo_id=index_repo_id,
+                allow_patterns=['*.faiss', '*.pkl'], 
+                repo_type='dataset',
+                local_dir='indexes/run/'
+            )
+            logger.info('Loading embedding model')
+            embed_instruction = "Represent the Hugging Face library documentation"
+            query_instruction = "Query the most relevant piece of information from the Hugging Face documentation"
+            embedding_model = HuggingFaceInstructEmbeddings(
+                model_name=embedding_model_id,
+                embed_instruction=embed_instruction,
+                query_instruction=query_instruction
+            )
+            logger.info('Loading index')
+            self.knowledge_index = FAISS.load_local(f"./indexes/run/", embedding_model)
+
+
+    def get_answer(self, question: str, messages_context: str = '') -> Response:
+        """
+        Generate an answer to the specified question.
+
+        Args:
+            question (str): The question to be answered.
+            messages_context (str, optional): The context to be used for generating the answer. Defaults to ''.
+
+        Returns:
+            response (Response): The Response object containing the generated answer and the sources of information 
+            used to generate the response.
+        """
+
+        response = Response()
+        context = 'Give an answer that contains all the necessary information for the question.\n'
+        relevant_docs = ''
+        if self.use_messages_for_context and messages_context:
+            messages_context = f'\nPrevious questions and answers:\n{messages_context}'
+            context += messages_context
+        if self.use_docs_for_context:
+            logger.info(f'Retriving documents')
+            relevant_docs = self.knowledge_index.similarity_search(
+                query=messages_context+question,
+                k=self.num_relevant_docs
+            )
+            context += '\nExtracted documents:\n'
+            context += "".join([doc.page_content for doc in relevant_docs])
+            metadata = [doc.metadata for doc in relevant_docs]
+            response.set_sources(sources=[str(m['source']) for m in metadata])
+
+        logger.info(f'Running LLM chain')
+        answer = self.llm_chain.run(question=question, context=context)
+        response.set_answer(answer)
+        logger.info(f'Received answer')
+
+        if self.debug:
+            sep = '\n' + '-' * 100
+            logger.info(sep)
+            logger.info(f'messages_contex: {messages_context} {sep}')
+            logger.info(f'relevant_docs: {relevant_docs} {sep}')
+            sources_str = '\n'.join(response.get_sources())
+            logger.info(f"sources:\n{sources_str} {sep}")
+            logger.info(f'context len: {len(context)} {sep}')
+            logger.info(f'context: {context} {sep}')
+            logger.info(f'question len: {len(question)}')
+            logger.info(f'question: {question} {sep}')
+            logger.info(f'response: {response.get_answer()} {sep}')
+        return response

api/question_answering/response.py

@@ -0,0 +1,33 @@
+from typing import List
+
+
+class Response:
+    def __init__(self):
+        self.answer = ''
+        self.sources = []
+
+    def set_answer(self, answer: str) -> None:
+        self.answer = answer
+
+    def set_sources(self, sources: List) -> None:
+        self.sources = list(set([str(s) for s in sources]))
+
+    def get_sources(self) -> List[str]:
+        return self.sources
+
+    def get_sources_as_text(self) -> str:
+        if not self.sources:
+            return ''
+        sources_text = '\n\nSources:'
+        for i, (source) in enumerate(self.sources):
+            sources_text += f'\n [{i+1}] {source}'
+        return sources_text
+
+    def get_answer(self, include_sources: bool = False) -> str:
+        answer = self.answer
+        if include_sources:
+            answer += self.get_sources_as_text()
+        return answer
+
+    def __str__(self):
+        return self.get_answer(include_sources=True)

app.py

@@ -0,0 +1,33 @@
+import gradio as gr
+from api.config import Config
+from api.logger import logger
+from api.question_answering import QAModel
+
+
+config = Config()
+model = QAModel(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    use_docs_for_context=config.use_docs_for_context,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    debug=config.debug
+)
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox()
+    clear = gr.ClearButton([msg, chatbot])
+
+    def respond(message, chat_history):
+        context = "".join(f"User: {msg} \nBot:{bot_msg}\n" for msg, bot_msg in chat_history)
+        logger.info(f"Context: {context}")
+        answer = model.get_answer(message, context)
+        bot_message = answer.get_response() + answer.get_sources_as_text() + "\n"
+        chat_history.append((message, bot_message))
+        return "", chat_history
+
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+
+demo.launch(share=False)

bot/__init__.py

@@ -0,0 +1,6 @@
+from dotenv import load_dotenv
+from bot.logger import setup_logger
+
+
+setup_logger()
+load_dotenv(dotenv_path='config/bot/.env')

bot/__main__.py

@@ -0,0 +1,19 @@
+from bot.config import Config
+from bot.logger import logger
+from bot.discord_client import DiscordClient
+
+
+def main():
+    logger.info('Starting Application...')
+    config = Config()
+    client = DiscordClient(
+        qa_service_url=config.qa_service_url,
+        num_last_messages=config.num_last_messages,
+        use_names_in_context=config.use_names_in_context,
+        enable_commands=config.enable_commands,
+        debug=config.debug
+    )
+    client.run(config.discord_token)
+
+if __name__ == '__main__':
+    main()

bot/config.py

@@ -0,0 +1,43 @@
+import os
+from dataclasses import dataclass, asdict
+from typing import Dict, Union
+from bot.logger import logger
+
+
+def get_env(env_name: str, default = None) -> str:
+    env = os.getenv(env_name)
+    if not env:
+        if default:
+            logger.warning(
+                f'Environment variable {env_name} not found.' \
+                f'Using the default value: {default}.'
+            )
+            return default
+        else:
+            raise ValueError(f'Cannot parse: {env_name}')
+    else:
+        return env
+
+
+@dataclass
+class Config:
+    discord_token: str = get_env('DISCORD_TOKEN')
+    qa_service_url: str = get_env('QA_SERVICE_URL')
+    add_sources_to_response: bool = eval(get_env('ADD_SOURCES_TO_RESPONSE', 'True'))
+    use_messages_in_context: bool = eval(get_env('USE_MESSEGES_IN_CONTEXT', 'True'))
+    num_last_messages: int = int(get_env('NUM_LAST_MESSAGES', 2))
+    use_names_in_context: bool = eval(get_env('USE_NAMES_IN_CONTEXT', 'False'))
+    enable_commands: bool = eval(get_env('ENABLE_COMMANDS', 'True'))
+    debug: bool = eval(get_env('DEBUG', 'True'))
+
+    def __post_init__(self):
+        # validate config
+        self.log()
+
+    def asdict(self) -> Dict:
+        return asdict(self)
+
+    def log(self) -> None:
+        logger.info('Config:')
+        for key, value in self.asdict().items():
+            logger.info(f'{key}: {value}')

bot/discord_client/__init__.py

@@ -0,0 +1 @@
+from .client import DiscordClient

bot/discord_client/client.py

@@ -0,0 +1,132 @@
+import json
+import requests
+from urllib.parse import quote
+import discord
+from typing import List
+
+from bot.logger import logger
+from bot.discord_client.utils import split_text_into_chunks
+
+
+class DiscordClient(discord.Client):
+    """
+    Discord Client class, used for interacting with a Discord server.
+
+    Args:
+        qa_service_url (str): The URL of the question answering service.
+        num_last_messages (int, optional): The number of previous messages to use as context for generating answers.
+        Defaults to 5.
+        use_names_in_context (bool, optional): Whether to include user names in the message context. Defaults to True.
+        enable_commands (bool, optional): Whether to enable commands for the bot. Defaults to True.
+
+    Attributes:
+        qa_service_url (str): The URL of the question answering service.
+        num_last_messages (int): The number of previous messages to use as context for generating answers.
+        use_names_in_context (bool): Whether to include user names in the message context.
+        enable_commands (bool): Whether to enable commands for the bot.
+        max_message_len (int): The maximum length of a message.
+        system_prompt (str): The system prompt to be used.
+
+    """
+    def __init__(
+        self,
+        qa_service_url: str,
+        num_last_messages: int = 5,
+        use_names_in_context: bool = True,
+        enable_commands: bool = True,
+        debug: bool = False
+    ):
+        logger.info('Initializing Discord client...')
+        intents = discord.Intents.all()
+        intents.message_content = True
+        super().__init__(intents=intents, command_prefix='!')
+
+        assert num_last_messages >= 1, \
+            'The number of last messages in context should be at least 1'
+
+        self.qa_service_url: str = qa_service_url
+        self.num_last_messages: int = num_last_messages
+        self.use_names_in_context: bool = use_names_in_context
+        self.enable_commands: bool = enable_commands
+        self.debug: bool = debug
+        self.min_messgae_len: int = 1800
+        self.max_message_len: int = 2000
+
+
+    async def on_ready(self):
+        """
+        Callback function to be called when the client is ready.
+        """
+        logger.info('Successfully logged in as: {0.user}'.format(self))
+        await self.change_presence(activity=discord.Game(name='Chatting...'))
+
+
+    async def get_last_messages(self, message) -> List[str]:
+        """
+        Method to fetch recent messages from a message's channel.
+
+        Args:
+            message (Message): The discord Message object used to identify the channel.
+
+        Returns:
+            List[str]: Reversed list of recent messages from the channel,
+            excluding the input message. Messages may be prefixed with the author's name 
+            if `self.use_names_in_context` is True.
+        """
+        last_messages: List[str] = []
+        async for msg in message.channel.history(
+            limit=self.num_last_messages):
+            if self.use_names_in_context:
+                last_messages.append(f'{msg.author}: {msg.content}')
+            else:
+                last_messages.append(msg.content)
+        last_messages.reverse()
+        last_messages.pop() # remove last message from context
+        return last_messages
+
+
+    async def send_message(self, message, answer: str, sources: str):
+        chunks = split_text_into_chunks(
+            text=answer,
+            split_characters=[". ", ", ", "\n"],
+            min_size=self.min_messgae_len,
+            max_size=self.max_message_len
+        )
+        for chunk in chunks:
+            await message.channel.send(chunk)
+        await message.channel.send(sources)
+
+
+    async def on_message(self, message):
+        """
+        Callback function to be called when a message is received.
+
+        Args:
+            message (discord.Message): The received message.
+        """
+        if message.author == self.user:
+            return
+        if self.enable_commands and message.content.startswith('!'):
+            if message.content == '!clear':
+                await message.channel.purge()
+                return
+
+        last_messages = await self.get_last_messages(message)
+        context = '\n'.join(last_messages)
+
+        logger.info('Received message: {0.content}'.format(message))
+        question_encoded = quote(message.content, safe='')
+        context_encoded = quote(context, safe='')
+        url = \
+            f'{self.qa_service_url}/' \
+            f'?question={question_encoded}' \
+            f'?&messgages_context={context_encoded}'
+        response = requests.get(url)
+        response.raise_for_status()
+        response = json.loads(response.content)
+
+        logger.info('Sending response: {0}'.format(response))
+        try:
+            await self.send_message(message, response['answer'], response['sources'])
+        except Exception as e:
+            logger.error('Failed to send response: {0}'.format(e))

bot/discord_client/utils.py

@@ -0,0 +1,54 @@
+from typing import List
+
+
+def find_max_split_index(text: str, char: str) -> int:
+    char_idx = text.rfind(char)
+    if char_idx > 0:
+        # If a character is found, return the index after the splitting character
+        split_idx = char_idx + len(char)
+        if split_idx >= len(text):
+            return len(text)
+        else:
+            return split_idx
+    return -1
+
+
+def find_max_split_index_from_sequence(text: str, split_characters: List[str]) -> int:
+    split_index = max((
+        find_max_split_index(text, sequence)
+        for sequence in split_characters
+    ), default=-1)
+    return split_index
+
+
+def split_text_into_chunks(
+    text: str,
+    split_characters: List[str] = [],
+    min_size: int = 20,
+    max_size: int = 250,
+    ) -> List[str]:
+
+    chunks = []
+    start_idx = 0
+    end_idx = max_size
+    text_len = len(text)
+    while start_idx < text_len:
+        search_chunk = text[start_idx+min_size:end_idx]
+        split_idx = find_max_split_index_from_sequence(
+            text=search_chunk,
+            split_characters=split_characters
+        )
+        # if no spliting element found, set the maximal size
+        if split_idx < 1:
+            split_idx = end_idx
+        # if found - offset it by the starting idx of the chunk
+        else:
+            split_idx += start_idx + min_size
+
+        chunk = text[start_idx:split_idx]
+        chunks.append(chunk)
+
+        start_idx = split_idx
+        end_idx = split_idx + max_size
+
+    return chunks

bot/logger.py

@@ -0,0 +1,14 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+def setup_logger() -> None:
+    """
+    Set up the logger.
+    """
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

config/api/.env.example

@@ -0,0 +1,9 @@
+HUGGINGFACEHUB_API_TOKEN={your-hf-token}
+QUESTION_ANSWERING_MODEL_ID={hf-question-answering-model-ID}
+EMBEDDING_MODEL_ID={hf-embedding-model-ID}
+INDEX_NAME=index
+USE_DOCS_FOR_CONTEXT=True
+ADD_SOURCES_TO_RESPONSE=True
+USE_MESSAGES_IN_CONTEXT=True
+NUM_RELEVANT_DOCS=3
+DEBUG=True

config/bot/.env.example

@@ -0,0 +1,7 @@
+DISCORD_TOKEN={your-bot-token}
+QA_SERVICE_URL=http://api:8000
+USE_MESSEGES_IN_CONTEXT=True
+NUM_LAST_MESSAGES=1
+USE_NAMES_IN_CONTEXT=False
+ENABLE_COMMANDS=True
+DEBUG=True

data/datasets/hf_repositories_urls.json

@@ -0,0 +1,20 @@
+{
+    "urls": [
+        "https://github.com/huggingface/transformers",
+        "https://github.com/huggingface/diffusers",
+        "https://github.com/huggingface/datasets",
+        "https://github.com/gradio-app/gradio",
+        "https://github.com/huggingface/huggingface_hub",
+        "https://github.com/huggingface/optimum",
+        "https://github.com/huggingface/tokenizers",
+        "https://github.com/huggingface/course",
+        "https://github.com/huggingface/deep-rl-class",
+        "https://github.com/huggingface/evaluate",
+        "https://github.com/huggingface/datasets-server",
+        "https://github.com/huggingface/simulate",
+        "https://github.com/huggingface/hub-docs",
+        "https://github.com/huggingface/pytorch-image-models",
+        "https://github.com/huggingface/safetensors",
+        "https://github.com/huggingface/hf-endpoints-documentation"
+    ]
+}

data/get_hugging_face_repositories.py

@@ -0,0 +1,34 @@
+import json
+import argparse
+import requests
+from typing import List
+
+
+def get_repositories_names(token):
+    url = f'https://api.github.com/orgs/huggingface/repos?per_page=1000'
+    headers = {'Authorization': f'token {token}'}
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        repos = json.loads(response.content)
+        repo_names = [
+            repo['full_name'] for repo in repos
+            if repo['stargazers_count'] >= 100
+        ]
+        return repo_names
+    else:
+        return 'Error: '+str(response.status_code)
+
+
+def save_repositories_urls(repositories_names: List[str], output_filename: str):
+    urls = ['https://github.com/'+repo_name for repo_name in repositories_names]
+    data = {"urls": urls}
+    with open(output_filename, 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--token', type=str)
+    args = parser.parse_args()
+    repositories = get_repositories_names(token=args.token)
+    save_repositories_urls(repositories, 'datasets/hf_repositories_urls_scraped.json')

data/hugging_face_docs_dataset.py

@@ -0,0 +1,190 @@
+import glob
+import json
+import os
+import re
+import subprocess
+from typing import List
+
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+from markdown import markdown
+import nbformat
+from nbconvert import MarkdownExporter
+from nbconvert.preprocessors import Preprocessor, ClearOutputPreprocessor
+from tqdm import tqdm
+
+
+VALIDATE_URLS = False
+
+
+def download_repositories(repo_urls_file: str, repo_dir: str):
+    """
+    Downloads the Hugging Face repositories.
+    """
+    if not os.path.exists(repo_dir):
+        os.makedirs(repo_dir)
+    with open(repo_urls_file, "r") as f:
+        repositories_urls = json.load(f)["urls"]
+        print(f'Downloading {len(repositories_urls)} repositories')
+        for url in repositories_urls:
+            try:
+                subprocess.run(["git", "clone", url], cwd=repo_dir)
+            except subprocess.CalledProcessError as e:
+                print("Command failed with error:", e.stderr)
+
+
+class EmptyCellPreprocessor(Preprocessor):
+    def preprocess_cell(self, cell, resources, index):
+        if cell.source.strip() == '':
+            cell.source = ''
+            cell.cell_type = 'raw'
+        return cell, resources
+
+
+def convert_notebook_to_txt(filename: str):
+    """
+    Converts a notebook to a markdown file.
+    """
+    with open(filename) as f:
+        notebook = nbformat.read(f, as_version=4)
+    # id validation error fix
+    for cell in notebook['cells']:
+        cell['id'] = str(cell['id'])
+
+    clear_output = ClearOutputPreprocessor()
+    notebook, resources = clear_output.preprocess(notebook, {})
+
+    exporter = MarkdownExporter()
+    exporter.register_preprocessor(EmptyCellPreprocessor, enabled=True)
+    output_notebook_text, resources = exporter.from_notebook_node(notebook)
+
+    new_filename = filename.replace('.ipynb', '_ipynb.txt')
+    with open(new_filename, 'w') as f:
+        f.write(output_notebook_text)
+    return new_filename
+
+
+def extract_files_from_directories(
+    repo_urls_file: str,
+    repo_dir: str,
+    docs_dir: str,
+    files_extensions: List[str]
+) -> None:
+
+    """
+    This function reads markdown and markdownx files from the repositories directory,
+    filters out non-English files, and adds the source GitHub URL as the first line of each file.
+    The resulting files are saved in the docs_dir.
+    """
+    languages = pd.read_csv("language-codes.csv").loc[:,"alpha2"].tolist()
+    languages.remove("en")
+
+    files = [
+        filename
+        for extension in files_extensions
+        for filename in glob.glob(repo_dir + f"**/*{extension}", recursive=True)
+    ]
+    print(f'Used extensions: {", ".join(files_extensions)}')
+    print(f'Found {len(files)} files')
+
+    repo_urls = []
+    with open(repo_urls_file, "r") as f:
+        repo_urls = json.load(f)["urls"]
+
+    # filter out the files that are not in english
+    filtered_files = []
+    for filename in files:
+        sep_file = filename.split("/")
+        for seq in sep_file:
+            if seq in languages:
+                break
+        else:
+            filtered_files.append(filename)
+    print(f'Found {len(filtered_files)} files in English')
+
+    # generate a GitHub URL for a file based on its name and a list of possible repository URLs
+    def get_github_url(filename: str, repo_urls: str, repo_dir: str) -> str:
+        source = filename.replace(repo_dir, '')
+        repo_name, file_path = source.split('/', 1)
+        repo_url_prefix = None
+        for repo_url in repo_urls:
+            if repo_name == repo_url.split('/')[-1]:
+                repo_url_prefix = repo_url
+                break
+        if not repo_url_prefix:
+            raise ValueError(f"Repo URL not found for {repo_name}")
+        url = f'{repo_url_prefix}/blob/main/{file_path}'
+        if VALIDATE_URLS:
+            try:
+                response = requests.get(url)
+                response.raise_for_status()
+            except:
+                print(f'filename: {filename}')
+                print(f'repo: {repo_name}, file: {file_path}')
+                print(f'url: {url}')
+                raise
+        return url
+
+    # creates a valid filename by replacing certain characters and removing the repo_dir path
+    def create_filename_from_path(filename: str, repo_dir: str) -> str:
+        filename = filename.replace(repo_dir, '')
+        chars_to_replace = ['/', '{', '}', '-', '.']
+        filename = ''.join(['_' if c in chars_to_replace else c for c in filename])
+        return filename
+
+    # copy the files with the source added in the first line
+    if not os.path.exists(docs_dir):
+        os.makedirs(docs_dir)
+    copied_files = []
+    for filename in tqdm(filtered_files):
+        source_url = get_github_url(filename, repo_urls, repo_dir)
+        data = f"source: {source_url}\n\n"
+        # convert jupyter notebooks to txt files
+        try:
+            if filename.endswith('.ipynb'):
+                filename = convert_notebook_to_txt(filename)
+            # rename and copy files
+            with open(filename, 'r') as f:
+                data += f.read()
+            output_filename = docs_dir + create_filename_from_path(filename, repo_dir)
+            with open(output_filename, 'w') as f:
+                f.write(data)
+            if not os.path.isfile(output_filename):
+                raise ValueError(f"Failed to create the output file: {output_filename}")
+            copied_files.append(output_filename)
+        except Exception as ex:
+            print(f'Failed to copy file {filename}: {ex}')
+
+    print(f'Successfully copied {len(set(copied_files))}/{len(filtered_files)} files')
+
+
+def markdown_cleaner(data: str):
+    """
+    Clean markdown text.
+
+    Args:
+        data (str): The markdown text to be cleaned.
+
+    Returns:
+        str: The cleaned markdown text.
+    """
+    soupped = BeautifulSoup(markdown(data), "html.parser")
+    raw_text = ''.join(soupped.findAll(string=True))
+    clean_text = re.sub(r"<!--.*?-->", "", raw_text, flags=re.DOTALL)
+    # remove any special tokens e.g <|endoftext|>
+    clean_text = re.sub(r"<\|endoftext\|>", "", clean_text, flags=re.DOTALL)
+    # discard non english text
+    clean_text = re.sub(r"[^a-zA-Z0-9\s]", "", clean_text, flags=re.DOTALL)
+    return "\n".join([t for t in clean_text.split("\n") if t])
+
+
+if __name__ == '__main__':
+    repo_urls_file = "./datasets/hf_repositories_urls.json"
+    repo_dir = "./datasets/huggingface_repositories/"
+    docs_dir = "./datasets/huggingface_docs/"
+    download_repositories(repo_urls_file, repo_dir)
+    extract_files_from_directories(
+        repo_urls_file, repo_dir, docs_dir,
+        files_extensions=['.md', '.mdx', '.ipynb']
+    )

data/indexer.ipynb

@@ -0,0 +1,226 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "from typing import List, Any\n",
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings\n",
+    "from langchain.document_loaders import TextLoader\n",
+    "from langchain.indexes import VectorstoreIndexCreator\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import FAISS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docs = []\n",
+    "metadata = []\n",
+    "for p in Path(\"./datasets/huggingface_docs/\").iterdir():\n",
+    "    if not p.is_dir():\n",
+    "        with open(p) as f:\n",
+    "            # the first line is the source of the text\n",
+    "            source = f.readline().strip().replace('source: ', '')\n",
+    "            docs.append(f.read())\n",
+    "            metadata.append({\"source\": source})\n",
+    "\n",
+    "print(f'number of documents: {len(docs)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text_splitter = CharacterTextSplitter(\n",
+    "    separator=\"\",\n",
+    "    chunk_size=812,\n",
+    "    chunk_overlap=100,\n",
+    "    length_function=len,\n",
+    ")\n",
+    "docs = text_splitter.create_documents(docs, metadata)\n",
+    "print(f'number of chunks: {len(docs)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = \"hkunlp/instructor-large\"\n",
+    "embed_instruction = \"Represent the Hugging Face library documentation\"\n",
+    "query_instruction = \"Query the most relevant piece of information from the Hugging Face documentation\"\n",
+    "\n",
+    "# embedding_model = HuggingFaceInstructEmbeddings(\n",
+    "#     model_name=model_name,\n",
+    "#     embed_instruction=embed_instruction,\n",
+    "#     query_instruction=query_instruction,\n",
+    "# )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class AverageInstructEmbeddings(HuggingFaceInstructEmbeddings):\n",
+    "    max_length: int = None\n",
+    "\n",
+    "    def __init__(self, max_length: int = 512, **kwargs: Any):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.max_length = max_length\n",
+    "        if self.max_length < 0:\n",
+    "            print('max_length is not specified, using model default max_seq_length')\n",
+    "\n",
+    "    def embed_documents(self, texts: List[str]) -> List[List[float]]:\n",
+    "        all_embeddings = []\n",
+    "        for text in tqdm(texts, desc=\"Embedding documents\"):\n",
+    "            if len(text) > self.max_length and self.max_length > -1:\n",
+    "                n_chunks = math.ceil(len(text)/self.max_length)\n",
+    "                chunks = [\n",
+    "                    text[i*self.max_length:(i+1)*self.max_length]\n",
+    "                    for i in range(n_chunks)\n",
+    "                ]\n",
+    "                instruction_pairs = [[self.embed_instruction, chunk] for chunk in chunks]\n",
+    "                chunk_embeddings = self.client.encode(instruction_pairs)\n",
+    "                avg_embedding = np.mean(chunk_embeddings, axis=0)\n",
+    "                all_embeddings.append(avg_embedding.tolist())\n",
+    "            else:\n",
+    "                instruction_pairs = [[self.embed_instruction, text]]\n",
+    "                embeddings = self.client.encode(instruction_pairs)\n",
+    "                all_embeddings.append(embeddings[0].tolist())\n",
+    "\n",
+    "        return all_embeddings\n",
+    "\n",
+    "\n",
+    "embedding_model = AverageInstructEmbeddings(  \n",
+    "    model_name=model_name,\n",
+    "    embed_instruction=embed_instruction,\n",
+    "    query_instruction=query_instruction,\n",
+    "    max_length=512,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = embedding_model.embed_documents(texts=[d.page_content for d in docs[:10]])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = FAISS.from_documents(docs, embedding_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.save_local('../indexes/index-large-notebooks/')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = FAISS.load_local(f'../indexes/index-large-notebooks/', embedding_model)\n",
+    "docs = index.similarity_search(query='how to create a pipeline object?', k=5)\n",
+    "docs[0].page_content\n",
+    "docs[0].metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for index, doc in enumerate(docs, start=1):\n",
+    "    print(f\"\\n{'='*100}\\n\")\n",
+    "    print(f\"Document {index} of {len(docs)}\")\n",
+    "    print(\"Page Content:\")\n",
+    "    print(f\"\\n{'-'*100}\\n\")\n",
+    "    print(doc.page_content, '\\n')\n",
+    "    print(doc.metadata)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "index_name = 'index-large-notebooks'\n",
+    "\n",
+    "api = HfApi()\n",
+    "api.create_repo(\n",
+    "    repo_id=f'KonradSzafer/{index_name}',\n",
+    "    repo_type='dataset',\n",
+    "    private=False,\n",
+    "    exist_ok=True\n",
+    ")\n",
+    "api.upload_folder(\n",
+    "    folder_path=f'../indexes/{index_name}',\n",
+    "    repo_id=f'KonradSzafer/{index_name}',\n",
+    "    repo_type='dataset',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_qa_bot",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data/language-codes.csv

@@ -0,0 +1,190 @@
+alpha2,English
+aa,Afar
+ab,Abkhazian
+ae,Avestan
+af,Afrikaans
+ak,Akan
+am,Amharic
+an,Aragonese
+ar,Arabic
+as,Assamese
+av,Avaric
+ay,Aymara
+az,Azerbaijani
+ba,Bashkir
+be,Belarusian
+bg,Bulgarian
+bh,Bihari languages
+bi,Bislama
+bm,Bambara
+bn,Bengali
+bo,Tibetan
+br,Breton
+bs,Bosnian
+ca,Catalan; Valencian
+ce,Chechen
+ch,Chamorro
+co,Corsican
+cr,Cree
+cs,Czech
+cu,Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic
+cv,Chuvash
+cy,Welsh
+da,Danish
+de,German
+dv,Divehi; Dhivehi; Maldivian
+dz,Dzongkha
+ee,Ewe
+el,"Greek, Modern (1453-)"
+en,English
+eo,Esperanto
+es,Spanish; Castilian
+et,Estonian
+eu,Basque
+fa,Persian
+ff,Fulah
+fi,Finnish
+fj,Fijian
+fo,Faroese
+fr,French
+fy,Western Frisian
+ga,Irish
+gd,Gaelic; Scottish Gaelic
+gj,Gujarati
+gl,Galician
+gn,Guarani
+gu,Gujarati
+gv,Manx
+ha,Hausa
+hd,Hindi
+he,Hebrew
+hi,Hindi
+ho,Hiri Motu
+hr,Croatian
+ht,Haitian; Haitian Creole
+hu,Hungarian
+hy,Armenian
+hz,Herero
+ia,Interlingua (International Auxiliary Language Association)
+id,Indonesian
+ie,Interlingue; Occidental
+ig,Igbo
+ii,Sichuan Yi; Nuosu
+ik,Inupiaq
+io,Ido
+is,Icelandic
+it,Italian
+iu,Inuktitut
+ja,Japanese
+jv,Javanese
+ka,Georgian
+kg,Kongo
+ki,Kikuyu; Gikuyu
+kj,Kuanyama; Kwanyama
+kk,Kazakh
+kl,Kalaallisut; Greenlandic
+km,Central Khmer
+kn,Kannada
+ko,Korean
+kr,Kanuri
+ks,Kashmiri
+ku,Kurdish
+kv,Komi
+kw,Cornish
+ky,Kirghiz; Kyrgyz
+la,Latin
+lb,Luxembourgish; Letzeburgesch
+lg,Ganda
+li,Limburgan; Limburger; Limburgish
+ln,Lingala
+lo,Lao
+lt,Lithuanian
+lu,Luba-Katanga
+lv,Latvian
+mg,Malagasy
+mh,Marshallese
+mi,Maori
+mk,Macedonian
+ml,Malayalam
+mn,Mongolian
+mr,Marathi
+ms,Malay
+mt,Maltese
+my,Burmese
+na,Nauru
+nb,"Bokmål, Norwegian; Norwegian Bokmål"
+nd,"Ndebele, North; North Ndebele"
+ne,Nepali
+ng,Ndonga
+nl,Dutch; Flemish
+nn,"Norwegian Nynorsk; Nynorsk, Norwegian"
+no,Norwegian
+nr,"Ndebele, South; South Ndebele"
+nv,Navajo; Navaho
+ny,Chichewa; Chewa; Nyanja
+oc,Occitan (post 1500)
+oj,Ojibwa
+om,Oromo
+or,Oriya
+os,Ossetian; Ossetic
+pa,Panjabi; Punjabi
+pi,Pali
+pl,Polish
+ps,Pushto; Pashto
+pt,Portuguese
+qu,Quechua
+rm,Romansh
+rn,Rundi
+ro,Romanian; Moldavian; Moldovan
+ru,Russian
+rw,Kinyarwanda
+sa,Sanskrit
+sc,Sardinian
+sd,Sindhi
+se,Northern Sami
+sg,Sango
+si,Sinhala; Sinhalese
+sk,Slovak
+sl,Slovenian
+sm,Samoan
+sn,Shona
+so,Somali
+sq,Albanian
+sr,Serbian
+ss,Swati
+st,"Sotho, Southern"
+su,Sundanese
+sv,Swedish
+sw,Swahili
+ta,Tamil
+te,Telugu
+tg,Tajik
+th,Thai
+ti,Tigrinya
+tk,Turkmen
+tl,Tagalog
+tn,Tswana
+to,Tonga (Tonga Islands)
+tr,Turkish
+ts,Tsonga
+tt,Tatar
+tw,Twi
+ty,Tahitian
+ug,Uighur; Uyghur
+uk,Ukrainian
+ur,Urdu
+uz,Uzbek
+ve,Venda
+vi,Vietnamese
+vo,Volapük
+wa,Walloon
+wo,Wolof
+xh,Xhosa
+yi,Yiddish
+yo,Yoruba
+za,Zhuang; Chuang
+zh,Chinese; General
+zh-CN,Chinese; Simplified
+zh-TW,Chinese; Traditional
+zh-hans,Chinese; Simplified
+zu,Zulu

data/scrapers/stack_overflow_scraper.py

@@ -0,0 +1,91 @@
+import re
+import csv
+import time
+import requests
+from typing import List
+import pandas as pd
+from tqdm import tqdm
+from bs4 import BeautifulSoup
+
+
+def scrape_question_with_answers(question_url: str) -> List[str]:
+    url = 'https://stackoverflow.com/' + question_url
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+
+    title = soup.find('title').text.replace(' - Stack Overflow', '')
+    question_div = soup.find('div', {'class': 'postcell post-layout--right'})
+    question = question_div.find('p').text
+    answers_div = soup.find('div', {'class': 'answercell post-layout--right'})
+    answer = answers_div.find('div', {'class': 's-prose js-post-body'}).text
+    return [title, question, answer, url]
+
+
+def scrape_questions_page(url: str, min_votes: int, min_answers: int) -> List[List[str]]:
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+    posts_summaries = soup.find_all('div', {'class':'s-post-summary js-post-summary'})
+
+    qa_data = []
+    for summary in posts_summaries:
+        stats_div = summary.find('div', {'class': 's-post-summary--stats'})
+        vote_div = stats_div.find('div', {
+            'class': 's-post-summary--stats-item s-post-summary--stats-item__emphasized',
+            'title': re.compile(r'^Score of \d+$')})
+        if vote_div:
+            vote_number = int(vote_div.find('span', {'class': 's-post-summary--stats-item-number'}).text)
+        else:
+            vote_number = 0
+        answer_div = stats_div.find('div', {
+            'class': 's-post-summary--stats-item',
+            'title': re.compile(r'^\d+ answers$')})
+        if answer_div:
+            answer_number = int(answer_div.find('span', {'class': 's-post-summary--stats-item-number'}).text)
+        else:
+            answer_number = 0
+
+        question_href = summary.find('a', {'class': 's-link'})['href']
+        if vote_number >= min_votes and answer_number >= min_answers:
+            try:
+                qa_data.append(scrape_question_with_answers(question_href))
+            except Exception as error:
+                print(error)
+
+        time.sleep(1.5)
+    return qa_data
+
+
+def crawl_and_save_qa(
+    filename: str,
+    base_url: str,
+    start_page: int,
+    n_pages: int=10,
+    min_votes: int=1,
+    min_answers: int=1
+):
+    with open(filename, 'a', newline='') as f:
+        writer = csv.writer(f)
+        if start_page == 1:
+            writer.writerow(['title', 'question', 'answer', 'url'])
+        for page_num in tqdm(range(start_page, start_page+n_pages)):
+            page_data = scrape_questions_page(
+                base_url.format(page_num),
+                min_votes,
+                min_answers
+            )
+            if page_data:
+                for qa_data in page_data:
+                    writer.writerow(qa_data)
+
+
+if __name__ == '__main__':
+    filename = '../datasets/stackoverflow_linux.csv'
+    url = 'https://stackoverflow.com/questions/tagged/linux?tab=votes&page={}&pagesize=15'
+    crawl_and_save_qa(
+        filename=filename,
+        base_url=url,
+        start_page=21,
+        n_pages=10,
+        min_votes=1,
+        min_answers=1
+    )

data/stackoverflow_python_dataset.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from datasets import load_dataset
+from bs4 import BeautifulSoup
+
+
+def preprocess_dataset():
+    """
+    Preprocesses the 'koutch/stackoverflow_python' dataset.
+
+    Returns:
+        datasets.arrow_dataset.Dataset: The preprocessed dataset.
+    """
+    dataset = load_dataset('koutch/stackoverflow_python', split='train')
+    dataset = dataset.filter(
+        lambda example:
+            example['question_score'] > 100 and
+            example['answer_score'] > 5 and
+            datetime.strptime(example['answer_date'], '%Y-%m-%dT%H:%M:%SZ').year > 2010
+        )
+
+    def html2text(example):
+        soup = BeautifulSoup(example, 'html.parser')
+        return ''.join(soup.findAll(string=True))
+
+    def transforms(example):
+        example['answer'] = html2text(example['answer_body'])
+        example['question'] = html2text(example['question_body'])
+        return example
+
+    dataset = dataset.map(lambda example: transforms(example))
+    dataset = dataset.remove_columns([
+        'question_score', 'question_date', 'question_id',
+        'answer_date', 'answer_id', 'answer_score', 'tags',
+        'question_body', 'answer_body'
+    ])
+    return dataset
+
+
+def show_info(dataset):
+    """
+    Print information about the dataset.
+
+    Args:
+        dataset (datasets.arrow_dataset.Dataset): The dataset.
+    """
+    print(dataset.info, '\n')
+    print(f'dataset len: {len(dataset)}')
+    print(f"example question: {dataset[0]['question']}")
+    print(f"example answer: {dataset[0]['answer']}")
+
+
+if __name__ == '__main__':
+    dataset = preprocess_dataset()
+    dataset.push_to_hub('KonradSzafer/stackoverflow_python_preprocessed', private=False)
+    show_info(dataset)

data/upload_csv_dataset.py

@@ -0,0 +1,24 @@
+import sys
+import pandas as pd
+from datasets import Dataset, DatasetDict
+from sklearn.model_selection import train_test_split
+
+
+
+def main():
+    dataset_name = sys.argv[1]
+    test_size = float(sys.argv[2]) if len(sys.argv) > 2 else 0.1
+    print(f'dataset: {dataset_name}, test size: {test_size}')
+
+    filename = f'datasets/{dataset_name}.csv'
+    df = pd.read_csv(filename)
+    dataset = Dataset.from_pandas(df)
+    train_dataset, test_dataset = train_test_split(dataset, test_size=test_size)
+    train_dataset = Dataset.from_dict(train_dataset)
+    test_dataset = Dataset.from_dict(test_dataset)
+    dataset_dict = DatasetDict({'train': train_dataset, 'test': test_dataset})
+    dataset_dict.push_to_hub(f'KonradSzafer/{dataset_name}', private=False)
+
+
+if __name__ == '__main__':
+    main()