add analysis script

eval_analysis.py

@@ -0,0 +1,299 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import json
+from pathlib import Path
+
+# Set style
+plt.style.use('ggplot')
+sns.set_palette("Set2")
+plt.rcParams['figure.figsize'] = (12, 8)
+
+# Load the data
+results_csv = "results/summary_20250421_230054.csv"
+results_json = "results/results_20250421_230054.json"
+
+df = pd.read_csv(results_csv)
+
+# Extract category from description if not already available
+def extract_category(row):
+    """
+    Determines the category of an image based on its description or existing category.
+    
+    Args:
+        row: A pandas DataFrame row containing 'category' and 'description' fields
+        
+    Returns:
+        str: The determined category ('fashion', 'landscape', 'abstract', or 'unknown')
+    """
+    if pd.notna(row['category']) and row['category'] != 'unknown':
+        return row['category']
+    
+    # Try to extract from description
+    desc = row['description'].lower()
+    if any(keyword in desc for keyword in ['coat', 'pants', 'shirt', 'dress', 'scarf', 'shoes']):
+        return 'fashion'
+    elif any(keyword in desc for keyword in ['forest', 'beach', 'mountain', 'ocean', 'lake', 'sky']):
+        return 'landscape'
+    elif any(keyword in desc for keyword in ['rectangle', 'circle', 'triangle', 'shape', 'spiral']):
+        return 'abstract'
+    else:
+        return 'unknown'
+
+# Clean the data
+df['category'] = df.apply(extract_category, axis=1)
+df['generation_time'] = pd.to_numeric(df['generation_time'], errors='coerce')
+
+# 1. Model Performance Comparison
+def plot_model_comparison():
+    """
+    Creates boxplots comparing model performance across three metrics:
+    VQA score, aesthetic score, and fidelity score.
+    
+    Saves the resulting plot to 'results/model_comparison.png'.
+    """
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+    
+    metrics = ['vqa_score', 'aesthetic_score', 'fidelity_score']
+    titles = ['VQA Score', 'Aesthetic Score', 'Fidelity Score']
+    
+    for i, (metric, title) in enumerate(zip(metrics, titles)):
+        sns.boxplot(x='model', y=metric, data=df, ax=axes[i])
+        axes[i].set_title(f'{title} by Model')
+        axes[i].set_ylim([0, 1])
+    
+    plt.tight_layout()
+    plt.savefig('results/model_comparison.png')
+    plt.close()
+
+# 2. Category Performance Analysis
+def plot_category_performance():
+    """
+    Creates boxplots showing performance by category and model for three metrics:
+    VQA score, aesthetic score, and fidelity score.
+    
+    Saves the resulting plot to 'results/category_performance.png'.
+    """
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+    
+    metrics = ['vqa_score', 'aesthetic_score', 'fidelity_score']
+    titles = ['VQA Score', 'Aesthetic Score', 'Fidelity Score']
+    
+    for i, (metric, title) in enumerate(zip(metrics, titles)):
+        sns.boxplot(x='category', y=metric, hue='model', data=df, ax=axes[i])
+        axes[i].set_title(f'{title} by Category and Model')
+        axes[i].set_ylim([0, 1])
+        if i > 0:
+            axes[i].get_legend().remove()
+    
+    axes[0].legend(title='Model')
+    plt.tight_layout()
+    plt.savefig('results/category_performance.png')
+    plt.close()
+
+# 3. Generation Time Analysis
+def plot_generation_time():
+    """
+    Creates visualizations of generation time analysis:
+    1. A boxplot showing generation time by model
+    2. Scatter plots showing the relationship between generation time and quality metrics
+    
+    Saves the resulting plots to 'results/generation_time.png' and 'results/quality_vs_time.png'.
+    """
+    plt.figure(figsize=(10, 6))
+    sns.boxplot(x='model', y='generation_time', data=df)
+    plt.title('Generation Time by Model')
+    plt.ylabel('Time (seconds)')
+    plt.tight_layout()
+    plt.savefig('results/generation_time.png')
+    plt.close()
+    
+    # Generation time vs quality scatter plot
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+    
+    metrics = ['vqa_score', 'aesthetic_score', 'fidelity_score']
+    titles = ['VQA Score', 'Aesthetic Score', 'Fidelity Score']
+    
+    for i, (metric, title) in enumerate(zip(metrics, titles)):
+        for model, color in zip(df['model'].unique(), ['#1f77b4', '#ff7f0e']):
+            model_data = df[df['model'] == model]
+            axes[i].scatter(model_data['generation_time'], model_data[metric], 
+                          alpha=0.6, label=model, c=color)
+            
+        axes[i].set_title(f'{title} vs. Generation Time')
+        axes[i].set_xlabel('Generation Time (seconds)')
+        axes[i].set_ylabel(title)
+        axes[i].legend()
+    
+    plt.tight_layout()
+    plt.savefig('results/quality_vs_time.png')
+    plt.close()
+
+# 4. Description complexity vs performance
+def plot_complexity_performance():
+    """
+    Analyzes the relationship between description complexity (word count) and 
+    performance metrics, creating scatter plots with trend lines.
+    
+    Saves the resulting plot to 'results/complexity_performance.png'.
+    """
+    df['description_length'] = df['description'].str.len()
+    df['word_count'] = df['description'].str.split().str.len()
+    
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+    
+    metrics = ['vqa_score', 'aesthetic_score', 'fidelity_score']
+    titles = ['VQA Score', 'Aesthetic Score', 'Fidelity Score']
+    
+    for i, (metric, title) in enumerate(zip(metrics, titles)):
+        for model, color in zip(df['model'].unique(), ['#1f77b4', '#ff7f0e']):
+            model_data = df[df['model'] == model]
+            axes[i].scatter(model_data['word_count'], model_data[metric], 
+                          alpha=0.6, label=model, c=color)
+            
+            # Add trendline
+            z = np.polyfit(model_data['word_count'], model_data[metric], 1)
+            p = np.poly1d(z)
+            axes[i].plot(sorted(model_data['word_count']), p(sorted(model_data['word_count'])), 
+                       c=color, linestyle='--')
+            
+        axes[i].set_title(f'{title} vs. Description Complexity')
+        axes[i].set_xlabel('Word Count')
+        axes[i].set_ylabel(title)
+        axes[i].legend()
+    
+    plt.tight_layout()
+    plt.savefig('results/complexity_performance.png')
+    plt.close()
+
+# 5. Success and failure examples
+def analyze_best_worst_examples():
+    """
+    Identifies and prints the top 10 most successful and least successful generations
+    based on fidelity score.
+    
+    Creates directories for sample SVG and PNG files if they don't exist.
+    
+    Returns:
+        tuple: (success_df, failure_df) DataFrames containing the best and worst examples
+    """
+    # Create directory for result samples
+    Path("results/sample_svg").mkdir(exist_ok=True)
+    Path("results/sample_png").mkdir(exist_ok=True)
+    
+    # Load detailed results
+    with open(results_json, 'r') as f:
+        results_data = json.load(f)
+    
+    # Create success/failure dataframes
+    success_df = df.nlargest(10, 'fidelity_score')
+    failure_df = df.nsmallest(10, 'fidelity_score')
+    
+    # Print success examples
+    print("Top 10 Successful Generations:")
+    print(success_df[['model', 'description', 'vqa_score', 'aesthetic_score', 'fidelity_score']].to_string(index=False))
+    
+    # Print failure examples
+    print("\nTop 10 Failed Generations:")
+    print(failure_df[['model', 'description', 'vqa_score', 'aesthetic_score', 'fidelity_score']].to_string(index=False))
+    
+    return success_df, failure_df
+
+# 6. Summary statistics
+def print_summary_stats():
+    """
+    Calculates and prints summary statistics for model performance:
+    1. Overall stats by model (mean, std, min, max for each metric)
+    2. Performance by category and model
+    
+    Also creates a radar chart visualizing fidelity scores by category and model,
+    saved to 'results/category_radar.png'.
+    """
+    # Overall stats by model
+    model_stats = df.groupby('model').agg({
+        'vqa_score': ['mean', 'std', 'min', 'max'],
+        'aesthetic_score': ['mean', 'std', 'min', 'max'],
+        'fidelity_score': ['mean', 'std', 'min', 'max'],
+        'generation_time': ['mean', 'std', 'min', 'max']
+    })
+    
+    print("Overall Model Performance:")
+    print(model_stats)
+    
+    # Stats by category and model
+    category_stats = df.groupby(['model', 'category']).agg({
+        'vqa_score': 'mean',
+        'aesthetic_score': 'mean',
+        'fidelity_score': 'mean',
+        'generation_time': 'mean'
+    }).reset_index()
+    
+    print("\nPerformance by Category and Model:")
+    print(category_stats.to_string())
+    
+    # Create a radar chart for category performance
+    categories = category_stats['category'].unique()
+    models = category_stats['model'].unique()
+    
+    plt.figure(figsize=(10, 8))
+    angles = np.linspace(0, 2*np.pi, len(categories), endpoint=False).tolist()
+    angles += angles[:1]  # Close the loop
+    
+    ax = plt.subplot(111, polar=True)
+    
+    for model in models:
+        model_data = category_stats[category_stats['model'] == model]
+        values = []
+        for category in categories:
+            cat_data = model_data[model_data['category'] == category]
+            if not cat_data.empty:
+                values.append(cat_data['fidelity_score'].values[0])
+            else:
+                values.append(0)
+        values += values[:1]  # Close the loop
+        
+        ax.plot(angles, values, linewidth=2, label=model)
+        ax.fill(angles, values, alpha=0.25)
+    
+    ax.set_xticks(angles[:-1])
+    ax.set_xticklabels(categories)
+    ax.set_title('Fidelity Score by Category and Model')
+    ax.legend(loc='upper right')
+    
+    plt.tight_layout()
+    plt.savefig('results/category_radar.png')
+    plt.close()
+
+# Main analysis function
+def run_analysis():
+    """
+    Main function that runs the complete analysis pipeline:
+    1. Creates necessary directories
+    2. Generates all visualization plots
+    3. Prints summary statistics
+    4. Analyzes best and worst examples
+    
+    All results are saved to the 'results/' directory.
+    """
+    print("Starting analysis of evaluation results...")
+    
+    # Create plots directory if it doesn't exist
+    Path("results").mkdir(exist_ok=True)
+    
+    # Generate all plots
+    plot_model_comparison()
+    plot_category_performance()
+    plot_generation_time()
+    plot_complexity_performance()
+    
+    # Print summary statistics
+    print_summary_stats()
+    
+    # Analyze best and worst examples
+    success_df, failure_df = analyze_best_worst_examples()
+    
+    print("\nAnalysis complete. Visualizations saved to 'results/' directory.")
+
+if __name__ == "__main__":
+    run_analysis()