Upload 2 files

Generate Error Analysis Visualizations/updated_enhanced_error_analysis.py

@@ -0,0 +1,353 @@
+# enhanced_error_analysis.py
+import json
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import pandas as pd
+from typing import List, Dict, Any
+
+# Error categories with detailed descriptions
+ERROR_CATEGORIES = {
+    "multi_step": "Chain-of-thought reasoning failures in multi-step problems",
+    "percentage": "Percentage and ratio calculations",
+    "logic": "Logical reasoning and problem setup failures",
+    "unit_conversion": "Measurement and unit conversion errors"
+}
+
+# Define colors for each category
+ERROR_CATEGORY_COLORS = {
+    "multi_step": "red",
+    "percentage": "blue", 
+    "logic": "orange",
+    "unit_conversion": "purple"
+}
+
+def refined_classify_error_type(result: Dict[str, Any]) -> str:
+    """
+    More precise error classification that distinguishes between error types
+    """
+    question = result["question"].lower()
+    ground_truth = str(result["ground_truth"]).lower()
+    predicted = str(result["predicted_answer"]).lower()
+    
+    # Check for percentage problems
+    if any(term in question for term in ['%', 'percent', 'percentage']):
+        return "percentage"
+    
+    # Check for unit conversion issues - prioritize this over multi-step
+    if any(unit in question for unit in ['pound', 'pounds', 'ounce', 'ounces', 'gallon', 'gallons', 
+                                       'mile', 'miles', 'hour', 'hours', 'minute', 'minutes', 
+                                       'second', 'seconds', 'dollar', 'dollars', 'cent', 'cents']):
+        return "unit_conversion"
+    
+    # Check for multi-step problems (complex wording)
+    # Exclude simple arithmetic problems that happen to have "twice", "half", etc.
+    multi_step_indicators = [
+        len(question.split()) > 30,  # Longer questions are more complex
+        any(connector in question for connector in ['if', 'then', 'after', 'before', 'when', 'however', 'since']),
+        question.count('and') > 3,  # More connections indicate complexity
+        any(phrase in question for phrase in ['first', 'then', 'next', 'finally', 'after that']),
+        any(term in question for term in ['each', 'every', 'per', 'total', 'combined'])
+    ]
+    
+    # Additional check to exclude simple problems with basic operations
+    simple_arithmetic_indicators = [
+        question.count('+') > 0,
+        question.count('-') > 0,
+        question.count('*') > 0,
+        question.count('/') > 0,
+        question.count('times') > 0,
+        question.count('plus') > 0,
+        question.count('minus') > 0
+    ]
+    
+    # Only classify as multi-step if it has multiple complexity indicators
+    # but doesn't look like simple arithmetic
+    complexity_score = sum(multi_step_indicators)
+    if complexity_score >= 3 and sum(simple_arithmetic_indicators) <= 2:
+        return "multi_step"
+    
+    # Default to logic if other categories don't fit
+    return "logic"
+
+def load_results(filename: str = "few_shot_results_errors_only.json") -> List[Dict[str, Any]]:
+    """Load the evaluation results from JSON file"""
+    try:
+        with open(filename, 'r') as f:
+            data = json.load(f)
+            
+        # Handle both formats: full results and error-only results
+        if "results" in data:
+            results = data.get("results", [])
+            # For error-only files, ensure all are marked as incorrect
+            for result in results:
+                result["is_correct"] = False
+            return results
+        else:
+            # Assume it's a list of results directly
+            return data
+            
+    except FileNotFoundError:
+        print(f"Error: File {filename} not found.")
+        return []
+    except json.JSONDecodeError:
+        print(f"Error: Invalid JSON format in {filename}.")
+        return []
+
+def create_comprehensive_visualization(results: List[Dict], num_samples: int):
+    """Create a comprehensive visualization showing all errors with proper categorization"""
+    
+    # Categorize errors
+    errors = [r for r in results if not r.get("is_correct", True)]
+    error_categories = {category: [] for category in ERROR_CATEGORIES}
+    
+    for error in errors:
+        category = refined_classify_error_type(error)
+        error_categories[category].append(error)
+    
+    # Prepare data for visualization
+    categories = list(error_categories.keys())
+    counts = [len(error_categories[cat]) for cat in categories]
+    percentages = [count/len(errors)*100 if errors else 0 for count in counts]
+    
+    # Create detailed error list for scatter plot
+    error_details = []
+    for category, errors_list in error_categories.items():
+        for error in errors_list:
+            error_details.append({
+                "sample_index": error.get("index", 0),
+                "category": category,
+                "ground_truth": error["ground_truth"],
+                "predicted": error["predicted_answer"],
+                "question_preview": error["question"][:50] + "..." if len(error["question"]) > 50 else error["question"]
+            })
+    
+    error_df = pd.DataFrame(error_details)
+    
+    # Create visualization
+    fig = make_subplots(
+        rows=2, cols=2,
+        subplot_titles=(
+            'Error Category Distribution',
+            'Error Samples by Index',
+            'Ground Truth vs Predicted Values',
+            'Error Analysis Summary'
+        ),
+        specs=[[{"type": "pie"}, {"type": "scatter"}],
+               [{"type": "scatter"}, {"type": "table"}]]
+    )
+    
+    # Pie chart - Error distribution with specified colors
+    fig.add_trace(
+        go.Pie(
+            labels=categories,
+            values=counts,
+            hole=0.4,
+            textinfo='label+value+percent',
+            hoverinfo='label+value+percent',
+            name="Error Types",
+            marker=dict(colors=[ERROR_CATEGORY_COLORS[cat] for cat in categories])
+        ),
+        row=1, col=1
+    )
+    
+    # Scatter plot - Error samples by index
+    for category in categories:
+        category_errors = error_df[error_df['category'] == category]
+        if not category_errors.empty:
+            fig.add_trace(
+                go.Scatter(
+                    x=category_errors['sample_index'],
+                    y=[1] * len(category_errors),
+                    mode='markers',
+                    marker=dict(size=12, color=ERROR_CATEGORY_COLORS[category]),
+                    name=category,
+                    text=category_errors['question_preview'],
+                    hoverinfo='text+x+y+name'
+                ),
+                row=1, col=2
+            )
+    
+    # Scatter plot - Ground truth vs predicted
+    if not error_df.empty:
+        fig.add_trace(
+            go.Scatter(
+                x=error_df['ground_truth'].astype(float),
+                y=error_df['predicted'].astype(float),
+                mode='markers',
+                marker=dict(
+                    size=10,
+                    color=[ERROR_CATEGORY_COLORS[cat] for cat in error_df['category']],
+                    opacity=0.7
+                ),
+                text=error_df['category'] + ": Sample " + error_df['sample_index'].astype(str),
+                hoverinfo='text+x+y',
+                name='GT vs Predicted'
+            ),
+            row=2, col=1
+        )
+        
+        # Add ideal line
+        max_val = max(max(error_df['ground_truth'].astype(float)), max(error_df['predicted'].astype(float))) + 10
+        fig.add_trace(
+            go.Scatter(
+                x=[0, max_val],
+                y=[0, max_val],
+                mode='lines',
+                line=dict(dash='dash', color='gray'),
+                name='Ideal',
+                showlegend=False
+            ),
+            row=2, col=1
+        )
+    
+    # Table - Error summary
+    summary_data = []
+    for category in categories:
+        for error in error_categories[category]:
+            summary_data.append([
+                error.get("index", "N/A"),
+                category,
+                error["ground_truth"],
+                error["predicted_answer"],
+                "✓" if error["ground_truth"] == error["predicted_answer"] else "✗"
+            ])
+    
+    if summary_data:
+        fig.add_trace(
+            go.Table(
+                header=dict(values=['Sample', 'Category', 'Ground Truth', 'Predicted', 'Correct']),
+                cells=dict(values=[
+                    [row[0] for row in summary_data],
+                    [row[1] for row in summary_data],
+                    [row[2] for row in summary_data],
+                    [row[3] for row in summary_data],
+                    [row[4] for row in summary_data]
+                ]),
+                name='Error Details'
+            ),
+            row=2, col=2
+        )
+    
+    # Update layout
+    fig.update_layout(
+        title=f"Comprehensive Error Analysis - {num_samples} Samples ({len(errors)} Errors)",
+        height=1000,
+        width=1400,
+        showlegend=True
+    )
+    
+    fig.update_xaxes(title_text="Sample Index", row=1, col=2)
+    fig.update_yaxes(title_text="", row=1, col=2, showticklabels=False)
+    fig.update_xaxes(title_text="Ground Truth", row=2, col=1)
+    fig.update_yaxes(title_text="Predicted", row=2, col=1)
+    
+    return fig, error_categories
+
+def generate_detailed_error_report(error_categories: Dict, num_samples: int):
+    """Generate a detailed report with analysis of each error category"""
+    
+    total_errors = sum(len(errors) for errors in error_categories.values())
+    accuracy = (num_samples - total_errors) / num_samples * 100 if num_samples > 0 else 0
+    
+    report = ["# Detailed Error Analysis Report", ""]
+    report.append(f"**Total Samples**: {num_samples}")
+    report.append(f"**Total Errors**: {total_errors}")
+    report.append(f"**Overall Accuracy**: {accuracy:.1f}%")
+    report.append("")
+    
+    for category, errors in error_categories.items():
+        if errors:
+            report.append(f"## {category.upper()} Errors ({len(errors)} errors)")
+            report.append("")
+            
+            for i, error in enumerate(errors, 1):
+                report.append(f"### Error {i}: Sample {error.get('index', 'N/A')}")
+                report.append("**Question:**")
+                report.append(f"> {error['question']}")
+                report.append("")
+                report.append("**Ground Truth:**")
+                report.append(f"`{error['ground_truth']}`")
+                report.append("")
+                report.append("**Model Prediction:**")
+                report.append(f"`{error['predicted_answer']}`")
+                report.append("")
+                report.append("**Error Analysis:**")
+                report.append(analyze_specific_error(error, category))
+                report.append("")
+                report.append("**Suggested Improvement:**")
+                report.append(suggest_improvement(error, category))
+                report.append("---")
+                report.append("")
+    
+    return "\n".join(report)
+
+def analyze_specific_error(error: Dict, category: str) -> str:
+    """Provide specific analysis for each error"""
+    question = error["question"]
+    generated = error.get("generated_text", "")
+    
+    if category == "percentage":
+        return "Percentage calculation error - likely misunderstanding of percentage relationships or incorrect application of percentage formulas."
+    
+    elif category == "multi_step":
+        # Check for specific multi-step failure patterns
+        if "then" not in generated.lower() and "so" not in generated.lower():
+            return "Missing logical connectors - model failed to show step-by-step reasoning process."
+        elif generated.count('\n') < 3:
+            return "Insufficient step breakdown - model attempted to solve in too few steps."
+        else:
+            return "Complex multi-step reasoning failure - model understood individual steps but failed to combine them correctly."
+    
+    elif category == "unit_conversion":
+        return "Unit conversion error - likely misunderstanding of measurement units or incorrect conversion between units."
+    
+    return "General reasoning error - model struggled with the problem structure."
+
+def suggest_improvement(error: Dict, category: str) -> str:
+    """Provide specific improvement suggestions"""
+    if category == "percentage":
+        return "Train on more percentage word problems with varied contexts. Implement percentage-specific prompting strategies."
+    
+    elif category == "multi_step":
+        return "Use chain-of-thought fine-tuning. Break complex problems into sub-tasks. Add intermediate supervision during training."
+    
+    elif category == "unit_conversion":
+        return "Practice unit conversion problems with step-by-step solutions. Focus on measurement and currency conversion scenarios. Train on real-world unit conversion applications."
+    
+    return "General reasoning training with diverse problem types and increased context understanding."
+
+def main():
+    # Load results
+    results = load_results('few_shot_results_errors_only.json')
+    
+    if not results:
+        print("❌ No results found. Exiting.")
+        return
+    
+    num_samples = len(results)
+    print(f"🔍 Performing comprehensive error analysis on {num_samples} error samples...")
+    
+    fig, error_categories = create_comprehensive_visualization(results, num_samples)
+    
+    # Save visualization
+    fig.write_html("enhanced_error_analysis.html")
+    print("💾 Enhanced visualization saved to enhanced_error_analysis.html")
+    
+    # Generate detailed report
+    report = generate_detailed_error_report(error_categories, num_samples)
+    with open("detailed_error_report.md", "w") as f:
+        f.write(report)
+    print("📝 Detailed report saved to detailed_error_report.md")
+    
+    # Print summary
+    total_errors = sum(len(errors) for errors in error_categories.values())
+    print(f"\n📊 Error Summary:")
+    for category, errors in error_categories.items():
+        if errors:
+            percentage = len(errors)/total_errors*100 if total_errors > 0 else 0
+            print(f"   {category.upper()}: {len(errors)} errors ({percentage:.1f}%)")
+    
+    print(f"   Overall Accuracy: {((num_samples - total_errors) / num_samples * 100):.1f}%" if num_samples > 0 else "N/A")
+
+if __name__ == "__main__":
+    main()

Generate Error Analysis Visualizations/updated_error_analysis.py

@@ -0,0 +1,361 @@
+# updated_error_analysis.py
+import json
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import plotly.express as px
+import pandas as pd
+from typing import Dict, List, Any
+import argparse
+
+# Error categories with detailed descriptions
+ERROR_CATEGORIES = {
+    "multi_step": "Chain-of-thought reasoning failures in multi-step problems",
+    "percentage": "Percentage and ratio calculations",
+    "logic": "Logical reasoning and problem setup failures",
+    "unit_conversion": "Measurement and unit conversion errors"
+}
+
+def classify_error_type(result: Dict[str, Any]) -> str:
+    """
+    Classify the type of error based on the question, ground truth, and predicted answer
+    """
+    question = result["question"].lower()
+    ground_truth = str(result["ground_truth"]).lower()
+    predicted = str(result["predicted_answer"]).lower()
+    
+    # Check for percentage problems
+    if any(term in question for term in ['%', 'percent', 'percentage']):
+        return "percentage"
+    
+    # Check for unit conversion issues - prioritize this over multi-step
+    if any(unit in question for unit in ['pound', 'pounds', 'ounce', 'ounces', 'gallon', 'gallons', 
+                                       'mile', 'miles', 'hour', 'hours', 'minute', 'minutes', 
+                                       'second', 'seconds', 'dollar', 'dollars', 'cent', 'cents']):
+        return "unit_conversion"
+    
+    # Check for multi-step problems (complex wording)
+    # Exclude simple arithmetic problems that happen to have "twice", "half", etc.
+    multi_step_indicators = [
+        len(question.split()) > 30,  # Longer questions are more complex
+        any(connector in question for connector in ['if', 'then', 'after', 'before', 'when', 'however', 'since']),
+        question.count('and') > 3,  # More connections indicate complexity
+        any(phrase in question for phrase in ['first', 'then', 'next', 'finally', 'after that']),
+        any(term in question for term in ['each', 'every', 'per', 'total', 'combined'])
+    ]
+    
+    # Additional check to exclude simple problems with basic operations
+    simple_arithmetic_indicators = [
+        question.count('+') > 0,
+        question.count('-') > 0,
+        question.count('*') > 0,
+        question.count('/') > 0,
+        question.count('times') > 0,
+        question.count('plus') > 0,
+        question.count('minus') > 0
+    ]
+    
+    # Only classify as multi-step if it has multiple complexity indicators
+    # but doesn't look like simple arithmetic
+    complexity_score = sum(multi_step_indicators)
+    if complexity_score >= 3 and sum(simple_arithmetic_indicators) <= 2:
+        return "multi_step"
+    
+    # Default to logic if other categories don't fit
+    return "logic"
+
+def load_results(filename: str = "few_shot_results_errors_only.json") -> List[Dict[str, Any]]:
+    """Load the evaluation results from JSON file"""
+    try:
+        with open(filename, 'r') as f:
+            data = json.load(f)
+            
+        # Handle both formats: full results and error-only results
+        if "results" in data:
+            results = data.get("results", [])
+            # For error-only files, ensure all are marked as incorrect
+            for result in results:
+                result["is_correct"] = False
+            return results
+        else:
+            # Assume it's a list of results directly
+            return data
+            
+    except FileNotFoundError:
+        print(f"Error: File {filename} not found.")
+        return []
+    except json.JSONDecodeError:
+        print(f"Error: Invalid JSON format in {filename}.")
+        return []
+
+def analyze_errors(results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """Perform comprehensive error analysis"""
+    total_samples = len(results)
+    
+    # For error-only analysis, all samples are incorrect
+    correct_count = 0
+    incorrect_count = total_samples
+    
+    # Categorize errors
+    error_categories = {category: [] for category in ERROR_CATEGORIES}
+    error_categories["correct"] = []
+    
+    for result in results:
+        if result["is_correct"]:
+            error_categories["correct"].append(result)
+        else:
+            error_type = classify_error_type(result)
+            error_categories[error_type].append(result)
+    
+    # Calculate statistics
+    accuracy = (correct_count / total_samples) * 100 if total_samples > 0 else 0
+    
+    category_stats = {
+        "total_samples": total_samples,
+        "correct_count": correct_count,
+        "incorrect_count": incorrect_count,
+        "accuracy": accuracy,
+        "category_counts": {cat: len(errors) for cat, errors in error_categories.items()},
+        "category_percentages": {
+            cat: (len(errors) / total_samples * 100) if total_samples > 0 else 0
+            for cat, errors in error_categories.items()
+        },
+        "detailed_errors": error_categories
+    }
+    
+    return category_stats
+
+def create_visualizations(stats: Dict[str, Any], num_samples: int):
+    """Create interactive Plotly visualizations with specified color scheme"""
+    
+    # Prepare data for visualizations
+    categories = list(ERROR_CATEGORIES.keys()) + ["correct"]
+    counts = [stats["category_counts"].get(cat, 0) for cat in categories]
+    percentages = [stats["category_percentages"].get(cat, 0) for cat in categories]
+    
+    # Create subplots
+    fig = make_subplots(
+        rows=2, cols=2,
+        subplot_titles=(
+            f'Error Distribution (n={num_samples})',
+            'Error Category Breakdown',
+            'Error Percentage by Category',
+            'Sample Index vs Error Type'
+        ),
+        specs=[[{"type": "pie"}, {"type": "bar"}],
+               [{"type": "scatter"}, {"type": "scatter"}]]
+    )
+    
+    # Define color scheme
+    color_map = {
+        'multi_step': 'red',
+        'percentage': 'blue', 
+        'logic': 'orange',
+        'unit_conversion': 'purple',
+        'correct': 'green'
+    }
+    
+    # Pie chart - Error distribution
+    error_categories_pie = [cat for cat in categories if cat != "correct" and stats["category_counts"].get(cat, 0) > 0]
+    error_counts_pie = [stats["category_counts"].get(cat, 0) for cat in error_categories_pie]
+    pie_colors = [color_map.get(cat, 'gray') for cat in error_categories_pie]
+    
+    fig.add_trace(
+        go.Pie(
+            labels=error_categories_pie,
+            values=error_counts_pie,
+            name="Error Types",
+            hole=0.4,
+            textinfo='label+percent',
+            hoverinfo='label+value+percent',
+            marker=dict(colors=pie_colors),
+            showlegend=False
+        ),
+        row=1, col=1
+    )
+    
+    # Bar chart - Category counts
+    non_zero_categories = [cat for cat in categories if cat != "correct" and stats["category_counts"].get(cat, 0) > 0]
+    non_zero_counts = [stats["category_counts"].get(cat, 0) for cat in non_zero_categories]
+    non_zero_percentages = [stats["category_percentages"].get(cat, 0) for cat in non_zero_categories]
+    bar_colors = [color_map.get(cat, 'gray') for cat in non_zero_categories]
+    
+    fig.add_trace(
+        go.Bar(
+            x=non_zero_categories,
+            y=non_zero_counts,
+            name="Count by Category",
+            marker_color=bar_colors,
+            text=[f"{count}<br>{percent:.1f}%" for count, percent in zip(non_zero_counts, non_zero_percentages)],
+            textposition='auto',
+            hoverinfo='x+y'
+        ),
+        row=1, col=2
+    )
+    
+    # Scatter plot - Error percentage by category
+    error_df = pd.DataFrame({
+        'Category': categories,
+        'Percentage': percentages,
+        'Count': counts
+    })
+    error_df = error_df[error_df['Count'] > 0]  # Only show categories with samples
+    scatter_colors = [color_map.get(cat, 'gray') for cat in error_df['Category']]
+    
+    fig.add_trace(
+        go.Scatter(
+            x=error_df['Category'],
+            y=error_df['Percentage'],
+            mode='markers+text',
+            marker=dict(
+                size=error_df['Count']*2 + 10, 
+                color=scatter_colors,
+                opacity=0.8
+            ),
+            text=error_df['Count'],
+            textposition='middle center',
+            name='Error Percentage',
+            hoverinfo='text',
+            hovertext=[f"{cat}: {pct:.1f}% ({cnt} samples)" for cat, pct, cnt in 
+                      zip(error_df['Category'], error_df['Percentage'], error_df['Count'])]
+        ),
+        row=2, col=1
+    )
+    
+    # Scatter plot - Sample index vs error type
+    all_results = []
+    for category in ERROR_CATEGORIES:
+        for result in stats["detailed_errors"][category]:
+            all_results.append((result, category))
+    
+    # Sort by sample index
+    all_results.sort(key=lambda x: x[0].get("index", 0))
+    
+    sample_indices = [result[0].get("index", i+1) for i, result in enumerate(all_results)]
+    error_types = [result[1] for result in all_results]
+    error_colors = [color_map.get(error_type, 'gray') for error_type in error_types]
+    
+    fig.add_trace(
+        go.Scatter(
+            x=sample_indices,
+            y=[1] * len(all_results),  # All are errors, so y=1
+            mode='markers',
+            marker=dict(
+                color=error_colors,
+                size=12,
+                opacity=0.7
+            ),
+            name='Error Types by Sample',
+            hoverinfo='x+y+text',
+            hovertext=[f"Sample {result[0].get('index', 'N/A')}: {result[1]}" for result in all_results]
+        ),
+        row=2, col=2
+    )
+    
+    # Update layout
+    fig.update_layout(
+        title=f"Symbolic-Math-Qwen2.5-1.5B-LoRA Error Analysis (n={num_samples} errors)",
+        height=1000,
+        width=1200,
+        showlegend=False
+    )
+    
+    fig.update_xaxes(title_text="Error Categories", row=1, col=2)
+    fig.update_yaxes(title_text="Number of Errors", row=1, col=2)
+    fig.update_xaxes(title_text="Categories", row=2, col=1)
+    fig.update_yaxes(title_text="Percentage (%)", row=2, col=1)
+    fig.update_xaxes(title_text="Sample Index", row=2, col=2)
+    fig.update_yaxes(title_text="Error Type", row=2, col=2, tickvals=[1], ticktext=["Errors"])
+    
+    return fig
+
+def generate_detailed_report(stats: Dict[str, Any]):
+    """Generate a detailed text report of the analysis"""
+    report = []
+    
+    report.append("=" * 60)
+    report.append("SYMBOLIC-MATH-QWEN2.5-1.5B-LoRA ERROR ANALYSIS REPORT")
+    report.append("=" * 60)
+    report.append(f"Total Error Samples: {stats['total_samples']}")
+    report.append(f"Overall Accuracy: {stats['accuracy']:.2f}%")
+    report.append("")
+    
+    report.append("ERROR CATEGORY BREAKDOWN:")
+    report.append("-" * 40)
+    for category, count in stats['category_counts'].items():
+        if category != "correct" and count > 0:
+            percentage = stats['category_percentages'][category]
+            report.append(f"{category.upper():<20}: {count:>3} errors ({percentage:>5.1f}%)")
+    
+    report.append("")
+    report.append("RECOMMENDATIONS:")
+    report.append("-" * 40)
+    
+    # Generate recommendations based on error patterns
+    if stats['category_counts'].get('percentage', 0) > 0:
+        report.append("• Add percentage calculation training examples")
+        report.append("• Implement percentage-specific prompting strategies")
+        report.append("• Train on more percentage word problems with varied contexts")
+    
+    if stats['category_counts'].get('multi_step', 0) > 0:
+        report.append("• Focus on chain-of-thought reasoning training")
+        report.append("• Break down complex problems into sub-steps")
+        report.append("• Add intermediate supervision during training")
+    
+    if stats['category_counts'].get('logic', 0) > 0:
+        report.append("• Improve logical reasoning capabilities")
+        report.append("• Train on problem setup and structure understanding")
+        report.append("• General reasoning training with diverse problem types")
+    
+    if stats['category_counts'].get('unit_conversion', 0) > 0:
+        report.append("• Practice unit conversion problems")
+        report.append("• Focus on measurement and currency conversions")
+        report.append("• Train on real-world unit conversion scenarios")
+    
+    return "\n".join(report)
+
+def main():
+    parser = argparse.ArgumentParser(description="Analyze Symbolic-Math-Qwen2.5-1.5B-LoRA errors on GSM8K dataset")
+    parser.add_argument("--samples", type=int, default=16, help="Number of error samples analyzed")
+    parser.add_argument("--input", type=str, default="few_shot_results_errors_only.json", help="Input JSON file")
+    parser.add_argument("--output", type=str, default="error_analysis.html", help="Output HTML file")
+    
+    args = parser.parse_args()
+    
+    print(f"🔍 Analyzing {args.samples} error samples...")
+    print(f"📁 Loading results from {args.input}")
+    
+    # Load and analyze results
+    results = load_results(args.input)
+    if not results:
+        print("❌ No results found. Exiting.")
+        return
+    
+    # Limit to specified number of samples
+    results = results[:args.samples]
+    
+    print("📊 Performing error analysis...")
+    stats = analyze_errors(results)
+    
+    # Generate visualizations
+    print("📈 Creating visualizations...")
+    fig = create_visualizations(stats, args.samples)
+    
+    # Save visualizations
+    fig.write_html(args.output)
+    print(f"💾 Visualizations saved to {args.output}")
+    
+    # Generate and print detailed report
+    report = generate_detailed_report(stats)
+    print("\n" + report)
+    
+    # Save report to file
+    report_filename = f"error_analysis_report_{args.samples}_errors.txt"
+    with open(report_filename, 'w') as f:
+        f.write(report)
+    print(f"📝 Detailed report saved to {report_filename}")
+    
+    print(f"\n🎯 Error analysis complete! Found {stats['total_samples']} error samples")
+    print("👉 Open the HTML file in your browser to view interactive visualizations")
+
+if __name__ == "__main__":
+    main()