| { |
| "created_at_utc": "2026-07-07T10:29:44Z", |
| "source_model": "feyninc/pulpie-orange-small", |
| "llama_cpp_note": "EuroBertForTokenClassification patch exposes per-token classifier logits via llama-embedding --pooling none.", |
| "environment": { |
| "python": "Python 3.11.10", |
| "torch": "2.4.1+cu124", |
| "device": "cpu", |
| "device_note": "PyTorch CUDA fallback to CPU: RuntimeError: CUDA error: no kernel image is available for execution on the device\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", |
| "cuda_available": true |
| }, |
| "files": [ |
| { |
| "file": "pulpie-orange-small-F16.gguf", |
| "size_bytes": 431400864, |
| "size_mb": 431.4 |
| }, |
| { |
| "file": "pulpie-orange-small-Q2_K.gguf", |
| "size_bytes": 130177056, |
| "size_mb": 130.18 |
| }, |
| { |
| "file": "pulpie-orange-small-Q3_K_M.gguf", |
| "size_bytes": 143153344, |
| "size_mb": 143.15 |
| }, |
| { |
| "file": "pulpie-orange-small-Q4_K_M.gguf", |
| "size_bytes": 156968320, |
| "size_mb": 156.97 |
| }, |
| { |
| "file": "pulpie-orange-small-Q5_K_M.gguf", |
| "size_bytes": 168912448, |
| "size_mb": 168.91 |
| }, |
| { |
| "file": "pulpie-orange-small-Q6_K.gguf", |
| "size_bytes": 181603104, |
| "size_mb": 181.6 |
| }, |
| { |
| "file": "pulpie-orange-small-Q8_0.gguf", |
| "size_bytes": 232886080, |
| "size_mb": 232.89 |
| } |
| ], |
| "load_checks": { |
| "pulpie-orange-small-F16.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 3.8506942209787667, |
| "llama_embedding_elapsed_seconds": 2.693078241078183 |
| }, |
| "pulpie-orange-small-Q2_K.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.4174793811980635, |
| "llama_embedding_elapsed_seconds": 2.4536153939552605 |
| }, |
| "pulpie-orange-small-Q3_K_M.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.399886897066608, |
| "llama_embedding_elapsed_seconds": 2.485915848053992 |
| }, |
| "pulpie-orange-small-Q4_K_M.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.6905582370236516, |
| "llama_embedding_elapsed_seconds": 2.5811137680429965 |
| }, |
| "pulpie-orange-small-Q5_K_M.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.3842295249924064, |
| "llama_embedding_elapsed_seconds": 2.456760450033471 |
| }, |
| "pulpie-orange-small-Q6_K.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.419072214048356, |
| "llama_embedding_elapsed_seconds": 2.4600620470009744 |
| }, |
| "pulpie-orange-small-Q8_0.gguf": { |
| "llama_cli_single_turn_exit_code": 0, |
| "llama_cli_loaded": true, |
| "llama_cli_note": "llama-cli is a generation frontend; this encoder classifier has no causal LM logits.", |
| "llama_embedding_exit_code": 0, |
| "llama_embedding_output_rows": 2, |
| "llama_cli_elapsed_seconds": 2.568599973106757, |
| "llama_embedding_elapsed_seconds": 2.5757998879998922 |
| } |
| }, |
| "accuracy": { |
| "pulpie-orange-small-F16.gguf": { |
| "texts": { |
| "test": { |
| "max_abs_diff": 0.004654441717529156, |
| "mean_abs_diff": 0.0022773042905806784, |
| "token_count": 2, |
| "prediction_agreement": 1.0 |
| }, |
| "Hello world": { |
| "max_abs_diff": 0.010810604415893543, |
| "mean_abs_diff": 0.005295509957885747, |
| "token_count": 3, |
| "prediction_agreement": 1.0 |
| }, |
| "<html><body><h1>Title</h1><p>Main text.</p></body></html>": { |
| "max_abs_diff": 0.06980780208435067, |
| "mean_abs_diff": 0.030381664234144044, |
| "token_count": 22, |
| "prediction_agreement": 1.0 |
| } |
| }, |
| "max_abs_diff": 0.06980780208435067, |
| "mean_abs_diff": 0.012651492827536822 |
| }, |
| "pulpie-orange-small-Q8_0.gguf": { |
| "texts": { |
| "test": { |
| "max_abs_diff": 0.06683230650177002, |
| "mean_abs_diff": 0.04104299360065457, |
| "token_count": 2, |
| "prediction_agreement": 1.0 |
| }, |
| "Hello world": { |
| "max_abs_diff": 0.0720623857818603, |
| "mean_abs_diff": 0.059363839538574215, |
| "token_count": 3, |
| "prediction_agreement": 1.0 |
| }, |
| "<html><body><h1>Title</h1><p>Main text.</p></body></html>": { |
| "max_abs_diff": 0.057413787133789064, |
| "mean_abs_diff": 0.021704604984231428, |
| "token_count": 22, |
| "prediction_agreement": 1.0 |
| } |
| }, |
| "max_abs_diff": 0.0720623857818603, |
| "mean_abs_diff": 0.040703812707820065 |
| } |
| }, |
| "e2e_extraction": { |
| "pulpie-orange-small-F16.gguf": { |
| "ok": true, |
| "n_blocks": 4, |
| "n_main": 3, |
| "n_other": 1, |
| "markdown": "<html><body>\n <main><h1 _item_id=\"1\">Orange harvest report</h1>\n <p _item_id=\"2\">The main article explains how growers sort oranges after harvest.</p>\n <p _item_id=\"3\">It includes storage notes and quality checks for shipment.</p>\n </main></body></html>", |
| "labels": { |
| "1": "main", |
| "2": "main", |
| "3": "main", |
| "4": "other" |
| }, |
| "tokenization_roundtrip_ok": true, |
| "latency_seconds": 13.650275907013565 |
| }, |
| "pulpie-orange-small-Q8_0.gguf": { |
| "ok": true, |
| "n_blocks": 4, |
| "n_main": 3, |
| "n_other": 1, |
| "markdown": "<html><body>\n <main><h1 _item_id=\"1\">Orange harvest report</h1>\n <p _item_id=\"2\">The main article explains how growers sort oranges after harvest.</p>\n <p _item_id=\"3\">It includes storage notes and quality checks for shipment.</p>\n </main></body></html>", |
| "labels": { |
| "1": "main", |
| "2": "main", |
| "3": "main", |
| "4": "other" |
| }, |
| "tokenization_roundtrip_ok": true, |
| "latency_seconds": 2.537674251012504 |
| } |
| }, |
| "latency_seconds": { |
| "pytorch": { |
| "pulpie-orange-small-F16.gguf": [ |
| 0.04418429802171886, |
| 0.07586899190209806, |
| 0.0618554730899632 |
| ], |
| "pulpie-orange-small-Q8_0.gguf": [ |
| 0.027146564098075032, |
| 0.032363954000175, |
| 0.05264966515824199 |
| ] |
| }, |
| "gguf_subprocess": { |
| "pulpie-orange-small-F16.gguf": [ |
| 2.542310531018302, |
| 2.5629363812040538, |
| 2.5437233441043645 |
| ], |
| "pulpie-orange-small-Q8_0.gguf": [ |
| 2.990182199049741, |
| 2.4438916840590537, |
| 2.455025817034766 |
| ], |
| "pulpie-orange-small-Q2_K.gguf": [ |
| 37.14351637708023, |
| 2.4945668019354343, |
| 2.5103745961096138 |
| ], |
| "pulpie-orange-small-Q3_K_M.gguf": [ |
| 28.285755896009505, |
| 2.4585004050750285, |
| 2.3625288009643555 |
| ], |
| "pulpie-orange-small-Q4_K_M.gguf": [ |
| 18.12280774395913, |
| 2.4188850200735033, |
| 2.4106604028493166 |
| ], |
| "pulpie-orange-small-Q5_K_M.gguf": [ |
| 2.9326697760261595, |
| 2.4721172840800136, |
| 2.514459784142673 |
| ], |
| "pulpie-orange-small-Q6_K.gguf": [ |
| 2.9089490061160177, |
| 2.4514556501526386, |
| 2.4820459550246596 |
| ] |
| }, |
| "pytorch_baseline": [ |
| 0.11215555900707841, |
| 0.19677641009911895, |
| 0.11335471202619374, |
| 0.17645823606289923, |
| 0.11712144990451634 |
| ] |
| } |
| } |
