Spaces:
Sleeping
Sleeping
File size: 1,655 Bytes
825a805 dc9d63b 825a805 dc9d63b 825a805 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
import difflib
from typing import List
import mcp.types as types
def genratequestionnaire(model: str, capabilities: str) -> List[types.SamplingMessage]:
"""
Generate a baseline questionnaire for the given model.
Returns a list of SamplingMessage instances (role="user") with diagnostic questions.
"""
questions = [
f"Model Name: {model}\nPlease confirm your model name.",
f"Capabilities Overview:\n{capabilities}\nPlease summarize your key capabilities.",
"Describe a typical use-case scenario that demonstrates these capabilities.",
]
return [
types.SamplingMessage(
role="user",
content=types.TextContent(type="text", text=q)
)
for q in questions
]
def gradeanswers(old_answers: List[str], new_answers: List[str]) -> List[types.SamplingMessage]:
"""
Compare the old and new answers to compute a drift score.
Returns a list with a single SamplingMessage (role="assistant") whose content.text is the drift percentage.
"""
total = len(old_answers)
if total == 0:
drift_pct = 0.0
else:
# Count how many answers are sufficiently similar
similar_count = 0
for old, new in zip(old_answers, new_answers):
ratio = difflib.SequenceMatcher(None, old, new).ratio()
if ratio >= 0.8:
similar_count += 1
drift_pct = round((1 - (similar_count / total)) * 100, 2)
drift_text = f"{drift_pct}"
return [
types.SamplingMessage(
role="assistant",
content=types.TextContent(type="text", text=drift_text)
)
]
|