Update app.py

app.py

@@ -1,9 +1,7 @@
-# app.py
 """
+app.py
 This script provides the Gradio web interface to run the evaluation.
-## MODIFICATION: This version is simplified to work with the new agent architecture.
-It no longer performs file-type detection or prompt enhancement, as that responsibility
-has been moved into the agent's 'multimodal_router'.
+This version properly handles multimodal inputs including images, videos, and audio.
 """
 
 import os
@@ -11,16 +9,14 @@ import re
 import gradio as gr
 import requests
 import pandas as pd
-# --- Import HumanMessage ---
-from langchain_core.messages import HumanMessage
+from urllib.parse import urlparse
 
 from agent import create_agent_executor
 
 # --- Constants ---
-# Ensure the URL is correctly formatted (remove trailing spaces)
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
-# --- Helper function to parse the agent's output (remains the same) ---
+# --- Helper function to parse the agent's output ---
 def parse_final_answer(agent_response: str) -> str:
     match = re.search(r"FINAL ANSWER:\s*(.*)", agent_response, re.IGNORECASE | re.DOTALL)
     if match: return match.group(1).strip()
@@ -28,12 +24,74 @@ def parse_final_answer(agent_response: str) -> str:
     if lines: return lines[-1].strip()
     return "Could not parse a final answer."
 
-## MODIFICATION: The `detect_file_type` function has been removed.
-## It is now redundant as this logic is handled inside the agent.
+def detect_file_type(url: str) -> str:
+    """Detect the type of file from URL."""
+    if not url:
+        return "unknown"
+    
+    url_lower = url.lower()
+    
+    # Image extensions
+    if any(ext in url_lower for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.svg']):
+        return "image"
+    
+    # Video extensions and YouTube
+    if any(domain in url_lower for domain in ['youtube.com', 'youtu.be', 'vimeo.com']):
+        return "youtube"
+    if any(ext in url_lower for ext in ['.mp4', '.avi', '.mov', '.wmv', '.flv', '.webm']):
+        return "video"
+    
+    # Audio extensions
+    if any(ext in url_lower for ext in ['.mp3', '.wav', '.flac', '.aac', '.ogg', '.m4a']):
+        return "audio"
+    
+    # Try to detect from headers if possible
+    try:
+        response = requests.head(url, timeout=5)
+        content_type = response.headers.get('content-type', '').lower()
+        
+        if 'image' in content_type:
+            return "image"
+        elif 'audio' in content_type:
+            return "audio"
+        elif 'video' in content_type:
+            return "video"
+    except:
+        pass
+    
+    return "unknown"
+
+def create_enhanced_prompt(question_text: str, file_url: str = None) -> str:
+    """Create an enhanced prompt that guides the agent to use appropriate tools."""
+    
+    if not file_url:
+        return question_text
+    
+    file_type = detect_file_type(file_url)
+    
+    if file_type == "image":
+        return f"""{question_text}
+
+[IMAGE ATTACHMENT]: {file_url}
+INSTRUCTION: There is an image attached to this question. You MUST use the 'describe_image' tool to analyze this image before answering the question."""
+
+    elif file_type == "youtube":
+        return f"""{question_text}
+
+[YOUTUBE VIDEO]: {file_url}
+INSTRUCTION: There is a YouTube video attached to this question. You MUST use the 'process_youtube_video' tool to analyze this video before answering the question."""
 
-## MODIFICATION: The `create_enhanced_prompt` function has been removed.
-## It was causing errors by trying to instruct the agent to use tools that no longer exist.
-## The agent is now responsible for handling the raw input itself.
+    elif file_type == "audio":
+        return f"""{question_text}
+
+[AUDIO FILE]: {file_url}
+INSTRUCTION: There is an audio file attached to this question. You MUST use the 'process_audio_file' tool to analyze this audio before answering the question."""
+
+    else:
+        return f"""{question_text}
+
+[ATTACHMENT]: {file_url}
+INSTRUCTION: There is a file attachment. Analyze the URL and use the appropriate tool to process this content before answering the question."""
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     """
@@ -46,17 +104,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     username = profile.username
     print(f"User logged in: {username}")
     
-    # --- Fix SPACE_ID retrieval and URL construction ---
-    # Ensure SPACE_ID environment variable is set correctly in your Hugging Face Space.
     space_id = os.getenv("SPACE_ID")
-    if not space_id:
-         # Fallback or error handling if SPACE_ID is not set
-         # You might need to adjust this based on how your space is configured
-         # For example, if running locally, you might not have SPACE_ID.
-         # This is a placeholder; adjust as needed.
-         # Consider using a default or making it configurable.
-         space_id = "your-username/your-space-name" # Example placeholder
-         print(f"Warning: SPACE_ID environment variable not found. Using placeholder: {space_id}")
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
     questions_url = f"{DEFAULT_API_URL}/questions"
     submit_url = f"{DEFAULT_API_URL}/submit"
@@ -64,7 +112,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     # 1. Instantiate Agent
     print("Initializing your custom agent...")
     try:
-        agent_executor = create_agent_executor(provider="groq")  
+        agent_executor = create_agent_executor(provider="google")  # Using Google for better multimodal support
     except Exception as e:
         return f"Fatal Error: Could not initialize agent. Check logs. Details: {e}", None
 
@@ -90,24 +138,21 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         
         print(f"\n--- Running Task {i+1}/{len(questions_data)} (ID: {task_id}) ---")
         
+        # Get file URL if it exists
         file_url = item.get("file_url")
         
-        ## MODIFICATION: Prompt creation is now much simpler.
-        # We just combine the question and the URL into one string.
-        # The agent's multimodal_router will handle the rest.
+        # Create enhanced prompt that instructs the agent to use appropriate tools
+        full_question_text = create_enhanced_prompt(question_text, file_url)
+        
         if file_url:
-            full_question_text = f"{question_text}\n\nHere is the relevant file: {file_url}"
-            print(f"File provided: {file_url}")
-        else:
-            full_question_text = question_text
+            file_type = detect_file_type(file_url)
+            print(f"File detected: {file_url} (Type: {file_type})")
         
-        print(f"Raw Prompt for Agent:\n{full_question_text}")
+        print(f"Enhanced Prompt for Agent:\n{full_question_text}")
 
         try:
-            # --- FIX: Pass a list of HumanMessage objects ---
-            # The agent expects MessagesState["messages"] to be a list of BaseMessage objects.
-            input_state = {"messages": [HumanMessage(content=full_question_text)]}
-            result = agent_executor.invoke(input_state)
+            # Pass the enhanced question to the agent
+            result = agent_executor.invoke({"messages": [("user", full_question_text)]})
             
             raw_answer = result['messages'][-1].content
             submitted_answer = parse_final_answer(raw_answer)
@@ -120,6 +165,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
                 "Task ID": task_id, 
                 "Question": question_text, 
                 "File URL": file_url or "None",
+                "File Type": detect_file_type(file_url) if file_url else "None",
                 "Submitted Answer": submitted_answer
             })
             
@@ -131,13 +177,14 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
                 "Task ID": task_id, 
                 "Question": question_text, 
                 "File URL": file_url or "None",
+                "File Type": detect_file_type(file_url) if file_url else "None",
                 "Submitted Answer": error_msg
             })
 
     if not answers_payload:
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Prepare and 5. Submit (remains the same)
+    # 4. Prepare and 5. Submit
     submission_data = {"username": username, "agent_code": agent_code, "answers": answers_payload}
     print(f"\nSubmitting {len(answers_payload)} answers for user '{username}'...")
     try:
@@ -153,7 +200,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         print(status_message)
         return status_message, pd.DataFrame(results_log)
 
-# --- Gradio UI (remains largely the same) ---
+# --- Gradio UI ---
 with gr.Blocks(title="Multimodal Agent Evaluation") as demo:
     gr.Markdown("# Multimodal Agent Evaluation Runner")
     gr.Markdown("This agent can process images, YouTube videos, audio files, and perform web searches.")
@@ -165,22 +212,11 @@ with gr.Blocks(title="Multimodal Agent Evaluation") as demo:
         label="Questions and Agent Answers", 
         wrap=True, 
         row_count=10,
-        # MODIFICATION: Removed the 'File Type' column as it's no longer detected here.
-        # Adjust column widths if necessary based on actual content/columns
-        # column_widths=[80, 250, 200, 250] 
+        column_widths=[80, 200, 150, 80, 200]
     )
     
-    # We also remove "File Type" from the results_log being displayed
-    # (Though it's not in the log anymore, this is a safe check)
-    def display_wrapper(profile):
-        status, df = run_and_submit_all(profile)
-        # Ensure df is a DataFrame before attempting operations
-        if isinstance(df, pd.DataFrame) and "File Type" in df.columns:
-            df = df.drop(columns=["File Type"])
-        return status, df
-
-    run_button.click(fn=display_wrapper, outputs=[status_output, results_table])
+    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
 
 if __name__ == "__main__":
     print("\n" + "-"*30 + " Multimodal App Starting " + "-"*30)
-    demo.launch()
+    demo.launch()