{
    "$schema": "http://json-schema.org/draft-07/schema#",
    "version": "0.1.0",
    "type": "object",
    "description": "Schema for storing and validating LLMs evaluation data, including model configuration, prompts, instances, Output, and evaluation metrics",
    "required": [
        "schema_version",
        "evaluation_id",
        "retrieved_timestamp",
        "source_data",
        "source_metadata",
        "model_info",
        "evaluation_results"
    ],
    "additionalProperties": false,
    "properties": {
        "schema_version": {
            "type": "string",
            "description": "Version of the schema used for this evaluation data"
        },
        "evaluation_id": {
            "type": "string",
            "description": "Unique identifier for this specific evaluation run. Use eval_name/model_id/retrieved_timestamp format"
        },
        "retrieved_timestamp": {
            "type": "string",
            "description": "Timestamp for when this record was created - using Unix Epoch time format"
        },
        "source_data": {
            "description": "Source of dataset used for evaluation. There are two options supported: HuggingFace dataset or url for other data source.",
            "oneOf": [
                {
                    "type": "array",
                    "description": "URLs for the source of the evaluation data",
                    "items": {
                        "type": "string"
                    }
                },
                {
                    "type": "object",
                    "description": "Details about HuggingFace dataset used for evaluation",
                    "required": [
                        "dataset_name"
                    ],
                    "properties": {
                        "dataset_name": {
                            "type": "string",
                            "description": "Name of the source dataset"
                        },
                        "hf_repo": {
                            "type": "string",
                            "description": "HuggingFace repository identifier"
                        },
                        "hf_split": {
                            "type": "string",
                            "description": "One of train, val or test."
                        },
                        "samples_number": {
                            "type": "integer",
                            "description": "Number of samples in the dataset"
                        },
                        "sample_ids": {
                            "type": "array",
                            "description": "Array of sample ids used for evaluation",
                            "items": {
                                "type": ["integer", "string"]
                            }
                        },
                        "additional_details": {
                            "type": "object",
                            "description": "Additional dataset info parameters",
                            "additionalProperties": true
                        }
                    }
                }
            ]
        },
        "source_metadata": {
            "type": "object",
            "description": "Metadata about the source of the leaderboard data",
            "required": [
                "source_type",
                "source_organization_name",
                "evaluator_relationship"
            ],
            "properties": {
                "source_name": {
                    "type": "string",
                    "description": "Name of the source (e.g. title of the source leaderboard or name of the platform used for the evaluation)."
                },
                "source_type": {
                    "type": "string",
                    "enum": [
                        "documentation",
                        "evaluation_run"
                    ],
                    "description": "Whether the data comes from a direct evaluation run or from documentation"
                },
                "source_organization_name": {
                    "type": "string",
                    "description": "Name of the organization that provides the data"
                },
                "source_organization_url": {
                    "type": "string",
                    "description": "URL for the organization that provides the data"
                },
                "source_organization_logo_url": {
                    "type": "string",
                    "description": "URL for the Logo for the organization that provides the data"
                },
                "evaluator_relationship": {
                    "type": "string",
                    "description": "Relationship between the evaluator and the model",
                    "enum": [
                        "first_party",
                        "third_party",
                        "collaborative",
                        "other"
                    ]
                }
            }
        },
        "model_info": {
            "type": "object",
            "description": "Complete model specification including basic information, technical configuration and inference settings",
            "required": [
                "name",
                "id"
            ],
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Model name provided by evaluation source"
                },
                "id": {
                    "type": "string",
                    "description": "Model name in HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct for models available on HuggingFace or openai/azure/gpt-4o-mini-2024-07-18 for closed API models)"
                },
                "developer": {
                    "type": "string",
                    "description": "Name of organization that provides the model (e.g. 'OpenAI')"
                },
                "inference_platform": {
                    "type": "string",
                    "description": "Name of inference platform which provides an access to models by API to run the evaluations or provides models weights to run them locally (e.g. HuggingFace, Bedrock, Together AI)"
                },
                "inference_engine": {
                    "type": "string",
                    "description": "Name of inference engine which provides an access to optimized models to use them for local evaluations (e.g. vLLM, Ollama)."
                },
                "additional_details": {
                    "type": "object",
                    "description": "Additional model configuration parameters",
                    "additionalProperties": true
                }
            }
        },
        "evaluation_results": {
            "type": "array",
            "description": "Array of evaluation results",
            "items": {
                "type": "object",
                "required": [
                    "evaluation_name",
                    "metric_config",
                    "score_details"
                ],
                "properties": {
                    "evaluation_name": {
                        "type": "string",
                        "description": "Name of the evaluation"
                    },
                    "evaluation_timestamp": {
                        "type": "string",
                        "description": "Timestamp for when the evaluations were run"
                    },
                    "metric_config": {
                        "type": "object",
                        "description": "Details about the metric",
                        "required": [
                            "lower_is_better"
                        ],
                        "properties": {
                            "evaluation_description": {
                                "type": "string",
                                "description": "Description of the evaluation"
                            },
                            "lower_is_better": {
                                "type": "boolean",
                                "description": "Whether a lower score is better"
                            },
                            "score_type": {
                                "type": "string",
                                "description": "Type of score",
                                "enum": [
                                    "binary",
                                    "continuous",
                                    "levels"
                                ]
                            },
                            "level_names": {
                                "type": "array",
                                "description": "Names of the score levels",
                                "items": {
                                    "type": "string"
                                }
                            },
                            "level_metadata": {
                                "type": "array",
                                "description": "Additional Description for each Score Level",
                                "items": {
                                    "type": "string"
                                }
                            },
                            "has_unknown_level": {
                                "type": "boolean",
                                "description": "Indicates whether there is an Unknown Level - if True, then a score of -1 will be treated as Unknown"
                            },
                            "min_score": {
                                "type": "number",
                                "description": "Minimum possible score for continuous metric"
                            },
                            "max_score": {
                                "type": "number",
                                "description": "Maximum possible score for continuous metric"
                            }
                        },
                        "if": {
                            "properties": {
                                "score_type": {
                                    "const": "levels"
                                }
                            }
                        },
                        "then": {
                            "required": [
                                "level_names",
                                "has_unknown_level"
                            ]
                        },
                        "else": {
                            "if": {
                                "properties": {
                                    "score_type": {
                                        "const": "continuous"
                                    }
                                }
                            },
                            "then": {
                                "required": [
                                    "min_score",
                                    "max_score"
                                ]
                            }
                        }
                    },
                    "score_details": {
                        "description": "The score for the evaluation and related details",
                        "required": [
                            "score"
                        ],
                        "properties": {
                            "score": {
                                "type": "number",
                                "description": "The score for the evaluation"
                            },
                            "details": {
                                "type": "object",
                                "description": "Any additional details about the score",
                                "additionalProperties": true
                            }
                        }
                    },
                    "detailed_evaluation_results_url": {
                        "type": "string",
                        "description": "Link to detailed evaluation data"
                    },
                    "generation_config": {
                        "type": "object",
                        "generation_args": {
                            "type": "object",
                            "description": "Parameters used to generate results - properties may vary by model type",
                            "properties": {
                                "temperature": {
                                    "type": [
                                        "null",
                                        "number"
                                    ],
                                    "description": "Sampling temperature"
                                },
                                "top_p": {
                                    "type": [
                                        "null",
                                        "number"
                                    ],
                                    "description": "Nucleus sampling parameter"
                                },
                                "top_k": {
                                    "type": [
                                        "null",
                                        "number"
                                    ],
                                    "description": "Top-k sampling parameter"
                                },
                                "max_tokens": {
                                    "type": "integer",
                                    "minimum": 1,
                                    "description": "Maximum number of tokens to generate"
                                },
                                "execution_command": {
                                    "type": "string",
                                    "description": "Command used to run the model to generate results"
                                },
                                "reasoning": {
                                    "type": "boolean",
                                    "description": "Whether reasoning orchain-of-thought was used to generate results"
                                }
                            },
                            "additionalProperties": true
                        },
                        "additional_details": {
                            "type": "string",
                            "description": "Additional details about how the results for this metric were generated."
                        }
                    }
                }
            }
        },
        "detailed_evaluation_results_per_samples": {
            "description": "Detailed eval results for all individual samples in the evaluation. This can be provided as source link or list of DetailedEvaluationResultsPerSample objects.",
            "anyOf": [
                {
                    "type": "string",
                    "description": "Link to detailed evaluation data for all samples"
                },
                {
                    "type": "array",
                    "description": "Array of evaluation results",
                    "items": {
                        "type": "object",
                        "required": [
                            "sample_id",
                            "input",
                            "ground_truth",
                            "response"
                        ],
                        "properties": {
                            "sample_id": {
                                "type": "string",
                                "description": "Simple sample ID"
                            },
                            "input": {
                                "type": "string",
                                "description": "Raw input for the model"
                            },
                            "prompt": {
                                "type": "string",
                                "description": "Full prompt for the model"
                            },
                            "ground_truth": {
                                "description": "Target response that may include one or multiple correct answers.",
                                "oneOf": [
                                    {
                                        "type": "string"
                                    },
                                    {
                                        "type": "array",
                                        "items": {
                                            "type": "string"
                                        }
                                    }
                                ]
                            },
                            "response": {
                                "type": "string",
                                "description": "Response from the model"
                            },
                            "choices": {
                                "description": "Either an array of possible responses (list of strings) or an array of string pairs [choice, response].",
                                "oneOf": [
                                    {
                                        "type": "array",
                                        "items": {
                                            "type": "string"
                                        }
                                    },
                                    {
                                        "type": "array",
                                        "items": {
                                            "type": "array",
                                            "items": [
                                                {
                                                    "type": "string"
                                                },
                                                {
                                                    "type": "string"
                                                }
                                            ],
                                            "minItems": 2,
                                            "maxItems": 2
                                        }
                                    }
                                ]
                            },
                            "full_logprobs": {
                                "type": "array",
                                "description": "Full log probabilities generated for this sample",
                                "items": {
                                    "type": "array",
                                    "items": {
                                        "type": "object",
                                        "required": [
                                            "token_id",
                                            "logprob",
                                            "decoded_token"
                                        ],
                                        "properties": {
                                            "token_id": {
                                                "type": "number",
                                                "description": "Id of token for which we keep its logprob"
                                            },
                                            "logprob": {
                                                "type": "number",
                                                "description": "Log probability of the token"
                                            },
                                            "decoded_token": {
                                                "type": "string",
                                                "description": "The decoded string representation of the token"
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            ]
        }
    }
}