# Imports import os import streamlit as st import requests from transformers import pipeline import openai # Suppressing all warnings import warnings warnings.filterwarnings("ignore") # Image-to-text def img2txt(url): print("Initializing captioning model...") captioning_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base") print("Generating text from the image...") text = captioning_model(url, max_new_tokens=20)[0]["generated_text"] print(text) return text # Text-to-story def txt2story(img_text): print("Initializing client...") client = openai.OpenAI( api_key=os.environ["TOGETHER_API_KEY"], base_url='https://api.together.xyz', ) messages = [ {"role": "system", "content": '''As an experienced short story writer, write story title and then create a meaningful story influenced by provided words. Ensure stories conclude positively within 100 words. Remember the story must end within 100 words''', "temperature": 1.8}, {"role": "user", "content": f"Here is input set of words: {img_text}", "temperature": 1.5}, ] print("Story...") chat_completion = client.chat.completions.create( messages=messages, model="togethercomputer/llama-2-70b-chat") print(chat_completion.choices[0].message.content) return chat_completion.choices[0].message.content # Text-to-speech def txt2speech(text): print("Initializing text-to-speech conversion...") API_URL = "https://api-inference.huggingface.co/models/espnet/kan-bayashi_ljspeech_vits" headers = {"Authorization": f"Bearer {os.environ['HUGGINGFACEHUB_API_TOKEN']}"} payloads = {'inputs': text} response = requests.post(API_URL, headers=headers, json=payloads) with open('audio_story.mp3', 'wb') as file: file.write(response.content) # Streamlit web app main function def main(): st.set_page_config(page_title="🎨 Image-to-Audio Story 🎧", page_icon="🖼️") st.title("Turn the Image into Audio Story") # Allows users to upload an image file uploaded_file = st.file_uploader("# 📷 Upload an image...", type=["jpg", "jpeg", "png"]) if uploaded_file is not None: # Reads and saves uploaded image file bytes_data = uploaded_file.read() with open("uploaded_image.jpg", "wb") as file: file.write(bytes_data) st.image(uploaded_file, caption='🖼️ Uploaded Image', use_column_width=True) # Initiates AI processing and story generation with st.spinner("## 🤖 AI is at Work! "): scenario = img2txt("uploaded_image.jpg") # Extracts text from the image story = txt2story(scenario) # Generates a story based on the image text txt2speech(story) # Converts the story to audio st.markdown("---") st.markdown("## 📜 Image Caption") st.write(scenario) st.markdown("---") st.markdown("## 📖 Story") st.write(story) st.markdown("---") st.markdown("## 🎧 Audio Story") st.audio("audio_story.mp3") if __name__ == '__main__': main() # Credits st.markdown("### Credits") st.caption(''' Made with ❤️ by @Aditya-Neural-Net-Ninja\n Utilizes Image-to-text, Text Generation, Text-to-speech Transformer Models\n Gratitude to Streamlit, 🤗 Spaces for Deployment & Hosting ''')