import gradio as gr import requests from PIL import Image import os import io import glob from transformers import BlipProcessor, BlipForConditionalGeneration import time from gradio_client import Client from huggingface_hub import HfApi import json from datetime import datetime token = os.getenv('HF_WRITE_TOKEN') blipper="Salesforce/blip-image-captioning-large" chatter="K00B404/transcript_image_generator" # Set your API endpoint and authorization details API_URL = "https://api-inference.huggingface.co/models/black-forest-labs/FLUX.1-schnell" headers = {"Authorization": f"Bearer {token}"} # Replace with your actual token timeout = 60 # seconds # Initialize the HuggingFace API api = HfApi(token=token) # Load BLIP model for image captioning processor = BlipProcessor.from_pretrained(blipper) model = BlipForConditionalGeneration.from_pretrained(blipper) # Initialize the API client for the chatbot chatbot_client = Client(chatter) PROFILES_FILENAME = "character_profiles.json" REPO_ID = "K00B404/Persona_from_Image" REPO_TYPE = "space" CHARACTERS_FOLDER = "characters" # Root folder for character images # Ensure characters folder exists if not os.path.exists(CHARACTERS_FOLDER): os.makedirs(CHARACTERS_FOLDER) def get_character_images(): """Get all image files from the characters folder""" image_files = [] # Get all PNG, JPG, JPEG, and WebP files for ext in ['png', 'jpg', 'jpeg', 'webp']: image_files.extend(glob.glob(f"{CHARACTERS_FOLDER}/*.{ext}")) image_files.extend(glob.glob(f"{CHARACTERS_FOLDER}/*.{ext.upper()}")) # Sort alphabetically image_files.sort() return image_files def load_profiles(): """Load profiles from HuggingFace space""" try: # Check if file exists in repo files = api.list_repo_files(repo_id=REPO_ID, repo_type=REPO_TYPE) if PROFILES_FILENAME not in files: # Create an empty profiles file if it doesn't exist empty_profiles = {"profiles": []} with open(f"/tmp/{PROFILES_FILENAME}", 'w') as f: json.dump(empty_profiles, f) api.upload_file( path_or_fileobj=f"/tmp/{PROFILES_FILENAME}", path_in_repo=PROFILES_FILENAME, repo_id=REPO_ID, repo_type=REPO_TYPE, ) return empty_profiles # Download the profiles file api.download_file( repo_id=REPO_ID, repo_type=REPO_TYPE, filename=PROFILES_FILENAME, local_dir="/tmp" ) # Read the profiles file with open(f"/tmp/{PROFILES_FILENAME}", 'r') as f: profiles = json.load(f) return profiles except Exception as e: print(f"Error loading profiles: {str(e)}") return {"profiles": []} def save_profiles(profiles): """Save profiles to HuggingFace space""" try: # Write profiles to temp file with open(f"/tmp/{PROFILES_FILENAME}", 'w') as f: json.dump(profiles, f) # Upload the profiles file api.upload_file( path_or_fileobj=f"/tmp/{PROFILES_FILENAME}", path_in_repo=PROFILES_FILENAME, repo_id=REPO_ID, repo_type=REPO_TYPE, ) return True except Exception as e: print(f"Error saving profiles: {str(e)}") return False def caption_to_persona(caption): """Convert a basic image caption into a character persona prompt""" persona = f"""You are {caption.replace('arafed image of ','a ').replace('arafed ','a ')} Your personality, speech patterns, knowledge, and behavior should reflect this description. When responding to users: 1. Stay in character at all times 2. Use speech patterns and vocabulary that would be natural for your character 3. Reference experiences, emotions, and perspectives that align with your character's background 4. Maintain a consistent personality throughout the conversation Additional context: Your responses should vary in length based on what would be natural for your character. Some characters might be terse while others might be more verbose.""" return persona def helper_llm(message, system_prompt, max_tokens=256, temperature=0.5, top_p=0.95): """Function to interact with the chatbot API using the generated persona""" try: # Call the API with the current message and system prompt (persona) response = chatbot_client.predict( message=message, system_message=system_prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, api_name="/chat" ) return response except Exception as e: return f"Error communicating with the chatbot API: {str(e)}" def generate_persona(img, min_len, max_len, persona_detail_level): # Process the image raw_image = Image.open(img).convert('RGB') # Resize image to 512x512 raw_image = raw_image.resize((256, 256), Image.Resampling.LANCZOS) inputs = processor(raw_image, return_tensors="pt") # Generate caption with specified length constraints start = time.time() out = model.generate(**inputs, min_length=min_len, max_length=max_len) caption = processor.decode(out[0], skip_special_tokens=True) # Enhance the caption based on detail level if persona_detail_level == "Basic": enhanced_caption = caption elif persona_detail_level == "Detailed": enhanced_caption = f"{caption} You have a distinct personality with unique mannerisms and speech patterns." else: # Comprehensive enhanced_caption = f"{caption} You have a complex backstory, rich emotional depth, unique perspectives, and distinctive speech patterns that set you apart." # Generate persona from caption persona = caption_to_persona(enhanced_caption) # Calculate processing time end = time.time() total_time = f"Processing time: {end - start:.2f} seconds" # dramaturg to mae a solid role for a actor from pragmatic description system_prompt="You are a Expert Dramaturg and your task is to use the input persona information and write a 'Role' description as compact instuctions for the actor" persona = helper_llm(persona, system_prompt=system_prompt) return caption, persona, total_time, img def chat_with_persona(message, history, system_message, max_tokens, temperature, top_p): """Function to interact with the chatbot API using the generated persona""" try: # Call the API with the current message and system prompt (persona) response = chatbot_client.predict( message=message, system_message=system_message, max_tokens=max_tokens, temperature=temperature, top_p=top_p, api_name="/chat" ) return response except Exception as e: return f"Error communicating with the chatbot API: {str(e)}" def generate_flux_image(final_prompt, is_negative, steps, cfg_scale, seed, strength): """ Generate an image using the FLUX model via Hugging Face's inference API. The function sends a POST request with the given payload and returns the image, along with the seed and prompt used. """ payload = { "inputs": final_prompt, "is_negative": is_negative, "steps": steps, "cfg_scale": cfg_scale, "seed": seed, "strength": strength } response = requests.post(API_URL, headers=headers, json=payload, timeout=timeout) if response.status_code != 200: print(f"Error: Failed to get image. Response status: {response.status_code}") print(f"Response content: {response.text}") if response.status_code == 503: raise gr.Error(f"{response.status_code} : The model is being loaded") raise gr.Error(f"{response.status_code}") try: image_bytes = response.content image = Image.open(io.BytesIO(image_bytes)) # Optionally save the image to a file (filename based on seed) output_path = f"./output_{seed}.png" image.save(output_path) print(f'\033[1mGeneration completed!\033[0m (Prompt: {final_prompt})') return output_path, str(seed), final_prompt except Exception as e: print(f"Error when trying to open the image: {e}") return None, None, None def process_selected_image(image_path): """Process a selected image from the gallery to generate a persona""" if not image_path: return "", "", "No image selected", None try: # Use default values for generation min_len = 50 max_len = 200 persona_detail_level = "Comprehensive" caption, persona, time_output, _ = generate_persona(image_path, min_len, max_len, persona_detail_level) return caption, persona, time_output, image_path except Exception as e: return "", "", f"Error processing image: {str(e)}", None # Create Gradio interface with tabs with gr.Blocks(title="Image Character Persona Generator") as iface: # Store state variables persona_state = gr.State("") profiles_state = gr.State(load_profiles()) selected_image_state = gr.State(None) with gr.Tabs(): # First tab: Persona Generator with gr.TabItem("Generate Persona"): gr.Markdown("# Image Character Persona Generator") gr.Markdown("Upload an image containing a character to generate an LLM persona based on that character.") with gr.Row(): with gr.Column(): input_image = gr.Image(type='filepath', label='Character Image') min_length = gr.Slider(label='Minimum Description Length', minimum=10, maximum=500, value=50, step=5) max_length = gr.Slider(label='Maximum Description Length', minimum=50, maximum=1000, value=200, step=10) detail_level = gr.Radio(["Basic", "Detailed", "Comprehensive"], label="Persona Detail Level", value="Comprehensive") submit_btn = gr.Button("Generate Character Persona") with gr.Column(): caption_output = gr.Textbox(label='Character Description (Base Caption)') persona_output = gr.Textbox(label='LLM Character Persona Prompt', lines=10) time_output = gr.Textbox(label='Processing Information') gr.Markdown(""" ## How to use this tool 1. Upload an image containing a character (real or fictional) 2. Adjust the sliders to control description length 3. Select detail level for the persona 4. Click "Generate Character Persona" 5. Switch to the "Test Persona" tab to chat with your character 6. create similar images inspired by the 'role' """) # Second tab: Test Character Chat with gr.TabItem("Test Persona"): gr.Markdown("# Test Your Character Persona") gr.Markdown("Chat with an AI using your generated character persona to see how it behaves.") with gr.Row(): with gr.Column(): system_prompt = gr.Textbox(label="Character Persona (System Prompt)", lines=8) with gr.Accordion("Advanced Settings", open=False): max_tokens = gr.Slider(label="Max Tokens", minimum=50, maximum=2048, value=512, step=1) temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.5, value=0.7, step=0.1) top_p = gr.Slider(label="Top P", minimum=0.1, maximum=1.0, value=0.95, step=0.05) with gr.Column(): chatbot = gr.Chatbot(label="Conversation with Character") msg = gr.Textbox(label="Your message") clear_btn = gr.Button("Clear Conversation") # Handle sending messages in the chat def respond(message, chat_history, system_message, max_tokens, temperature, top_p): if not message.strip(): return "", chat_history # Add user message to history chat_history.append((message, "")) # Get response from API bot_response = chat_with_persona(message, chat_history, system_message, max_tokens, temperature, top_p) # Update the last response in history chat_history[-1] = (message, bot_response) return "", chat_history # Clear chat history def clear_chat(): return [] # Connect message input to chat response msg.submit(respond, [msg, chatbot, system_prompt, max_tokens, temperature, top_p], [msg, chatbot]) clear_btn.click(clear_chat, outputs=chatbot) # New Tab 3: Flux Image Generation with gr.Tab("Flux Image Generation"): gr.Markdown("### Flux Image Generation") final_prompt = gr.Textbox(label="Prompt", lines=2, placeholder="Enter your prompt for Flux...") is_negative = gr.Checkbox(label="Use Negative Prompt", value=False) steps = gr.Slider(minimum=10, maximum=100, step=1, value=50, label="Steps") cfg_scale = gr.Slider(minimum=1, maximum=20, step=1, value=7, label="CFG Scale") seed = gr.Number(value=42, label="Seed") strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, value=0.8, label="Strength") generate_button = gr.Button("Generate Flux Image") output_image = gr.Image(label="Generated Image") output_seed = gr.Textbox(label="Seed Used") output_prompt = gr.Textbox(label="Prompt Used") generate_button.click( fn=generate_flux_image, inputs=[final_prompt, is_negative, steps, cfg_scale, seed, strength], outputs=[output_image, output_seed, output_prompt] ) # New Tab 4: Persona Manager with gr.Tab("Persona Manager"): gr.Markdown("### Character Persona Manager") gr.Markdown("Save, edit, and load character profiles to maintain a persistent library of characters.") with gr.Row(): with gr.Column(scale=1): # Character Gallery gr.Markdown("### Character Images") # Auto-load images from characters folder character_gallery = gr.Gallery( label="Available Character Images", elem_id="character_gallery", columns=3, object_fit="contain", height="auto" ) refresh_gallery_btn = gr.Button("Refresh Character Gallery") # Profile Management Section gr.Markdown("### Profile Management") profile_name = gr.Textbox(label="Profile Name", placeholder="Enter a name for this character profile") add_profile_btn = gr.Button("Save Current Profile") status_msg = gr.Textbox(label="Status", interactive=False) # Profile Selection Dropdown profile_dropdown = gr.Dropdown(label="Select Saved Profile", interactive=True) load_profile_btn = gr.Button("Load Selected Profile") update_profile_btn = gr.Button("Update Selected Profile") delete_profile_btn = gr.Button("Delete Selected Profile") refresh_profiles_btn = gr.Button("Refresh Profiles List") with gr.Column(scale=2): # Profile Preview preview_image = gr.Image(label="Character Image Preview", type="filepath") # Profile Editor profile_caption = gr.Textbox(label="Character Description", lines=3) profile_persona = gr.Textbox(label="Character Persona", lines=15) image_path_display = gr.Textbox(label="Image Path (Reference Only)", interactive=False) # Process Selected Image Button process_image_btn = gr.Button("Process Selected Image") # Copy to Chat Button copy_to_chat_btn = gr.Button("Use This Persona in Chat") # Modified function def refresh_character_gallery(): image_files = get_character_images() return image_files # Return just the list of image paths # Modified function def gallery_select(evt: gr.SelectData, gallery_images): if gallery_images and evt.index < len(gallery_images): selected_image = gallery_images[evt.index] return selected_image, selected_image return None, None # Function to handle gallery selection def gallery_select(evt: gr.SelectData, gallery_images): if gallery_images and evt.index < len(gallery_images): selected_image = gallery_images[evt.index] return selected_image, selected_image return None, None # Function to refresh the profiles dropdown def refresh_profiles_list(profiles_data): profiles = profiles_data["profiles"] return gr.Dropdown(choices=[p["name"] for p in profiles], value=None if not profiles else profiles[0]["name"]) # Function to save current profile def save_current_profile(name, caption, persona, image_path, profiles_data): if not name or not caption or not persona: return profiles_data, "Error: Please provide a name, caption, and persona", gr.Dropdown(choices=[p["name"] for p in profiles_data["profiles"]]) # Create new profile object new_profile = { "name": name, "caption": caption, "persona": persona, "image_path": image_path if image_path else "", "created_at": datetime.now().isoformat() } # Check if profile with this name already exists for i, profile in enumerate(profiles_data["profiles"]): if profile["name"] == name: # Update existing profile profiles_data["profiles"][i] = new_profile save_profiles(profiles_data) return profiles_data, f"Updated existing profile: {name}", gr.Dropdown(choices=[p["name"] for p in profiles_data["profiles"]], value=name) # Add new profile profiles_data["profiles"].append(new_profile) save_profiles(profiles_data) # Return updated profiles data and status message profile_names = [p["name"] for p in profiles_data["profiles"]] return profiles_data, f"Saved new profile: {name}", gr.Dropdown(choices=profile_names, value=name) # Function to load selected profile def load_selected_profile(profile_name, profiles_data): for profile in profiles_data["profiles"]: if profile["name"] == profile_name: image_path = profile["image_path"] if os.path.exists(profile["image_path"]) else None return profile["caption"], profile["persona"], profile["image_path"], image_path, "Profile loaded successfully", profile["persona"] return "", "", "", None, "Error: Profile not found", "" # Function to update selected profile def update_selected_profile(profile_name, caption, persona, image_path, profiles_data): if not profile_name: return profiles_data, "Error: No profile selected" for i, profile in enumerate(profiles_data["profiles"]): if profile["name"] == profile_name: # Update profile data profiles_data["profiles"][i]["caption"] = caption profiles_data["profiles"][i]["persona"] = persona if image_path and image_path != profile["image_path"]: profiles_data["profiles"][i]["image_path"] = image_path profiles_data["profiles"][i]["updated_at"] = datetime.now().isoformat() # Save updated profiles save_profiles(profiles_data) return profiles_data, f"Updated profile: {profile_name}" return profiles_data, "Error: Profile not found" # Function to delete selected profile def delete_selected_profile(profile_name, profiles_data): if not profile_name: return profiles_data, "Error: No profile selected", gr.Dropdown(choices=[p["name"] for p in profiles_data["profiles"]]) # Filter out the profile to delete profiles_data["profiles"] = [p for p in profiles_data["profiles"] if p["name"] != profile_name] # Save updated profiles save_profiles(profiles_data) # Return updated profiles data and status message profile_names = [p["name"] for p in profiles_data["profiles"]] return profiles_data, f"Deleted profile: {profile_name}", gr.Dropdown(choices=profile_names, value=None if not profile_names else profile_names[0]) # Function to use the current persona in chat def use_persona_in_chat(persona): return persona def initialize_interface(profiles_data): profiles_list = refresh_profiles_list(profiles_data) gallery = refresh_character_gallery() return profiles_list, gallery # Connect the UI elements with their functions refresh_gallery_btn.click( fn=refresh_character_gallery, outputs=[character_gallery] ) character_gallery.select( fn=gallery_select, inputs=[character_gallery], outputs=[selected_image_state, preview_image] ) process_image_btn.click( fn=process_selected_image, inputs=[selected_image_state], outputs=[profile_caption, profile_persona, status_msg, image_path_display] ) add_profile_btn.click( fn=save_current_profile, inputs=[profile_name, profile_caption, profile_persona, selected_image_state, profiles_state], outputs=[profiles_state, status_msg, profile_dropdown] ) load_profile_btn.click( fn=load_selected_profile, inputs=[profile_dropdown, profiles_state], outputs=[profile_caption, profile_persona, image_path_display, preview_image, status_msg, system_prompt] ) update_profile_btn.click( fn=update_selected_profile, inputs=[profile_dropdown, profile_caption, profile_persona, selected_image_state, profiles_state], outputs=[profiles_state, status_msg] ) delete_profile_btn.click( fn=delete_selected_profile, inputs=[profile_dropdown, profiles_state], outputs=[profiles_state, status_msg, profile_dropdown] ) refresh_profiles_btn.click( fn=refresh_profiles_list, inputs=[profiles_state], outputs=[profile_dropdown] ) copy_to_chat_btn.click( fn=use_persona_in_chat, inputs=[profile_persona], outputs=[system_prompt] ) # Create a dedicated initialization function def initialize_interface(profiles_data): profile_names = [p["name"] for p in profiles_data["profiles"]] if profiles_data and "profiles" in profiles_data else [] image_files = get_character_images() return profile_names, image_files # Return raw values, not components # Then use it in place of your lambda iface.load( fn=initialize_interface, inputs=[profiles_state], outputs=[profile_dropdown, character_gallery] ) # Function to update system prompt in Test tab when persona is generated def update_persona_state(caption, persona, time_output, img_path): return persona, persona # Connect the persona generator to update the system prompt submit_btn.click(fn=generate_persona, inputs=[input_image, min_length, max_length, detail_level], outputs=[caption_output, persona_output, time_output, selected_image_state]) # Update the system prompt in Test tab when persona is generated submit_btn.click(fn=update_persona_state, inputs=[caption_output, persona_output, time_output, input_image], outputs=[persona_state, system_prompt]) # Function to update profile fields when a new persona is generated def update_profile_fields(caption, persona, img_path): return caption, persona, img_path, img_path # Update the profile fields in Persona Manager tab when persona is generated submit_btn.click(fn=update_profile_fields, inputs=[caption_output, persona_output, input_image], outputs=[profile_caption, profile_persona, selected_image_state, preview_image]) # Launch the interface iface.launch()