From ddc96261366eb10d008ee5cc30b183815fb59f6b Mon Sep 17 00:00:00 2001 From: slobodanmargetic988 Date: Wed, 6 May 2026 14:28:25 +0200 Subject: [PATCH] =?UTF-8?q?publish:=2021=20run(s)=20=E2=80=94=2009d8fbde-0?= =?UTF-8?q?008-49bb-99da-03eeaca72be1,=201bf57c9a-fd7a-49aa-90de-cd1907b15?= =?UTF-8?q?ddd,=20212d6278-1b9b-45e9-8aae-7eed4d4ec822=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- catalogue.json | 12320 +++++++++++++++- .../metadata.json | 991 +- .../metadata.json | 22 +- .../metadata.json | 47 +- .../metadata.json | 343 +- .../metadata.json | 22 +- .../metadata.json | 22 +- .../metadata.json | 343 +- .../metadata.json | 1063 +- .../metadata.json | 52 +- .../metadata.json | 22 +- .../metadata.json | 109 +- .../metadata.json | 2441 ++- .../metadata.json | 1063 +- .../metadata.json | 1015 +- .../metadata.json | 22 +- .../metadata.json | 223 +- .../metadata.json | 22 +- .../metadata.json | 2119 ++- .../metadata.json | 903 +- .../metadata.json | 78 +- .../metadata.json | 1171 +- 22 files changed, 24099 insertions(+), 314 deletions(-) diff --git a/catalogue.json b/catalogue.json index d2e40fb..0efb908 100644 --- a/catalogue.json +++ b/catalogue.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", - "generated_at": "2026-05-06T08:04:27Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "generated_at": "2026-05-06T12:28:25Z", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "site_base_url": "https://benchmarks.weeyuga.com", "data_base_url": "https://benchmarks.weeyuga.com/data", "gitea_archive_url": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public", @@ -54,20 +54,1177 @@ "scatter" ] }, - "site_grade_count": 10, + "site_grade_count": 11, "site_grade_breakdown": { "flagship": 4, - "standard": 6, - "archive-only": 11 + "standard": 7, + "archive-only": 10, + "legacy": 104 + }, + "total_runs_in_archive": 125, + "by_model": { + "phi-4": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822" + ], + "gemma-4-26b-a4b": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822" + ], + "qwen3-30b-a3b": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822", + "fe4fa289-67a9-48e8-8a80-d5d88c875208" + ], + "qwen2-5-72b": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822" + ], + "gemma-4-26b-a4b-it": [ + "91751afd-068a-477b-8f40-6e1963f803f1" + ], + "qwen3-30b-a3b-iq2m": [ + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef" + ], + "gemma-4-e4b": [ + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499" + ], + "qwen3-30b-a3b-iq2xxs": [ + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44" + ], + "qwen3-5-9b": [ + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "09d8fbde-0008-49bb-99da-03eeaca72be1", + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check" + ], + "qwen3-14b": [ + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "granite-4-1-8b": [ + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "gemma-4-e4b-it": [ + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "hf-chinchilla-mix": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z" + ], + "qwen3-5-4b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:qwen3_5_4b" + ], + "qwen3-5-35b-a3b-uncensored": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen3-5-35b-a3b-iq2s": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen3-5-9b-q6k": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen3-5-2b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen3-5-0-8b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen3_5_0_8b" + ], + "qwen2-5-coder-14b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted" + ], + "qwen2-5-coder-3b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report" + ], + "qwen3-8b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen3-4b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "qwen2-5-3b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual" + ], + "qwen2-5-coder-1-5b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual" + ], + "qwen2-5-coder-0-5b": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual" + ], + "unknown": [ + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ], + "codellama-34b": [ + "legacy:vps81-telemetry:models:codellama34_16k" + ], + "codestral-22b": [ + "legacy:vps81-telemetry:models:codestral_32k" + ], + "llama3-2-3b": [ + "legacy:vps81-telemetry:models:llama3_2_3b" + ], + "phi-3-mini": [ + "legacy:vps81-telemetry:models:phi3" + ], + "phind-codellama-34b": [ + "legacy:vps81-telemetry:models:phind34_16k" + ], + "smollm2-135m": [ + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m" + ], + "smollm2-1-7b": [ + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m" + ], + "smollm2-360m": [ + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m" + ], + "qwen3-5-4b-mlx": [ + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b" + ], + "qwen2-5-14b": [ + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark" + ] + }, + "by_hardware": { + "vps50": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:codestral_32k", + "legacy:vps81-telemetry:models:llama3_2_3b", + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ], + "predator": [ + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", + "fe4fa289-67a9-48e8-8a80-d5d88c875208", + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "pavilion": [ + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark" + ], + "mac": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check" + ] + }, + "by_family": { + "gemma-phi-qwen2-5-qwen3": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822" + ], + "qwen3": [ + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07", + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", + "fe4fa289-67a9-48e8-8a80-d5d88c875208" + ], + "gemma": [ + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499" + ], + "qwen3-qwen3-5": [ + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d" + ], + "gemma-granite-qwen3-5": [ + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "chinchilla": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z" + ], + "qwen2-5-qwen2-5-coder-qwen3-qwen3-5": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "unknown": [ + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ], + "qwen3-5": [ + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check" + ], + "llama": [ + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:llama3_2_3b" + ], + "codestral": [ + "legacy:vps81-telemetry:models:codestral_32k" + ], + "phi": [ + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k" + ], + "qwen2-5-coder": [ + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual" + ], + "smollm2": [ + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m" + ], + "qwen2-5": [ + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual" + ] + }, + "by_task": { + "param-sweep": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07" + ], + "chat": [ + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", + "fe4fa289-67a9-48e8-8a80-d5d88c875208", + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "09d8fbde-0008-49bb-99da-03eeaca72be1", + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:codestral_32k", + "legacy:vps81-telemetry:models:llama3_2_3b", + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ], + "throughput": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z" + ] + }, + "by_engine": { + "llamacpp": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "86241185-24da-4f8e-b254-a60c9465e353", + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07", + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", + "fe4fa289-67a9-48e8-8a80-d5d88c875208", + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "mlx-or-hf": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z" + ], + "weeyuga": [ + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "ollama": [ + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:codestral_32k", + "legacy:vps81-telemetry:models:llama3_2_3b", + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ] + }, + "by_suite": { + "chinchilla": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z" + ], + "python-task-20q-v1": [ + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini" + ], + "python-task-10q-v2": [ + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini" + ], + "small-model-manual": [ + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ], + "hello-check": [ + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check" + ], + "vs-standard": [ + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b" + ], + "progress": [ + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress" + ], + "long-context": [ + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context" + ], + "concurrency": [ + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report" + ], + "coverage-audit": [ + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit" + ], + "ladder": [ + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview" + ], + "parallel": [ + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q" + ], + "mac-vs-vps": [ + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report" + ], + "benchmark-report": [ + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report" + ], + "python-overnight": [ + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini" + ] + }, + "by_grade": { + "standard": [ + "91751afd-068a-477b-8f40-6e1963f803f1", + "79afe366-1055-4e45-adf6-593864a530e8", + "e08a7051-a856-4053-b34b-561d7ce5e8cf", + "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", + "ad28cb95-5134-40b5-8514-c8a381f83d87", + "b37836bf-d1a2-4d6b-a732-aff89da1fa07", + "fe4fa289-67a9-48e8-8a80-d5d88c875208" + ], + "archive-only": [ + "86241185-24da-4f8e-b254-a60c9465e353", + "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", + "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", + "212d6278-1b9b-45e9-8aae-7eed4d4ec822", + "d1cff064-9141-48e0-b1d2-729b134e0543", + "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", + "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", + "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", + "ad057f5b-ed3f-4a95-a38e-361be310ffd6", + "ff1131ca-d021-4e06-8616-4b4cdb54e97e" + ], + "flagship": [ + "23066b38-ea9c-4dd3-b2f5-32912a67fce4", + "5fb2913d-6500-4ecf-9e97-d43f7dd61145", + "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", + "09d8fbde-0008-49bb-99da-03eeaca72be1" + ], + "legacy": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z", + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:codestral_32k", + "legacy:vps81-telemetry:models:llama3_2_3b", + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ] + }, + "by_source": { + "mac": [ + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b" + ], + "pavilion": [ + "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "legacy:pavilion:_template:windows_benchmark_template", + "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark" + ], + "vps81-telemetry": [ + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "legacy:vps81-telemetry:report-catalog", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "legacy:vps81-telemetry:models:codellama34_16k", + "legacy:vps81-telemetry:models:codestral_32k", + "legacy:vps81-telemetry:models:llama3_2_3b", + "legacy:vps81-telemetry:models:phi3", + "legacy:vps81-telemetry:models:phind34_16k", + "legacy:vps81-telemetry:models:qwen14_coder_32k", + "legacy:vps81-telemetry:models:qwen14_general_32k", + "legacy:vps81-telemetry:models:qwen2_5_3b", + "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "legacy:vps81-telemetry:models:qwen32_coder_32k", + "legacy:vps81-telemetry:models:qwen3_5_4b", + "legacy:vps81-telemetry:models:qwen3_5_9b", + "legacy:vps81-telemetry:models:riven_smolvlm", + "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "legacy:vps81-telemetry:2026-04-05-small-model-manual" + ] }, - "total_runs_in_archive": 21, "benchmarks": [ { "id": "91751afd-068a-477b-8f40-6e1963f803f1", "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T10:23:13Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -190,7 +1347,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -202,7 +1363,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -214,7 +1379,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -226,7 +1395,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b-it-q4km-cpu-ctx32k", @@ -238,7 +1411,37 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 2, @@ -267,7 +1470,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "212d6278-1b9b-45e9-8aae-7eed4d4ec822", @@ -283,7 +1487,7 @@ "title": "predator-a3b-ngl6-ctx-1 \u2014 qwen3 on predator", "date": "2026-05-05", "started_at": "2026-05-05T09:44:34Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -369,7 +1573,182 @@ "tokens_per_sec_mean": 5.42, "tokens_per_sec_p50": 5.35, "tokens_per_sec_p95": 5.91, - "tokens_per_sec_max": 5.93 + "tokens_per_sec_max": 5.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 88941.0, + "duration_ms_p50": 81140.0, + "tokens_per_sec_mean": 5.21 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 78787.89, + "duration_ms_p50": 70839.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22599.5, + "duration_ms_p50": 19202.0, + "tokens_per_sec_mean": 5.54, + "tokens_per_sec_p50": 5.54 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71981.0, + "duration_ms_p50": 73215.0, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 149398.0, + "duration_ms_p50": 146891.5, + "tokens_per_sec_mean": 5.23, + "tokens_per_sec_p50": 5.26 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1030.83, + "answer_chars_mean": 891.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 38170, + "tokens_per_sec": 5.187319884726224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15973, + "tokens_per_sec": 5.884930820760032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 13824, + "tokens_per_sec": 5.931712962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 22431, + "tokens_per_sec": 5.171414560206856, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 81140, + "tokens_per_sec": 5.385753019472516, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70839, + "tokens_per_sec": 5.533674953062579, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 60354, + "tokens_per_sec": 5.550584882526428, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 75591, + "tokens_per_sec": 5.5033006574856795, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 147513, + "tokens_per_sec": 5.063960464501434, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 157591, + "tokens_per_sec": 5.254107150789068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 146270, + "tokens_per_sec": 5.271074041156765, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 146218, + "tokens_per_sec": 5.313983230518815, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx96k", @@ -381,7 +1760,182 @@ "tokens_per_sec_mean": 5.43, "tokens_per_sec_p50": 5.38, "tokens_per_sec_p95": 5.78, - "tokens_per_sec_max": 5.94 + "tokens_per_sec_max": 5.94, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92845.33, + "duration_ms_p50": 110870.0, + "tokens_per_sec_mean": 5.26 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81762.11, + "duration_ms_p50": 88383.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 17637.5, + "duration_ms_p50": 17538.5, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.55 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 99966.0, + "duration_ms_p50": 110822.0, + "tokens_per_sec_mean": 5.4, + "tokens_per_sec_p50": 5.38 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 135995.25, + "duration_ms_p50": 151381.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.23 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1265.92, + "answer_chars_mean": 766.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 16242, + "tokens_per_sec": 5.294914419406477, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18234, + "tokens_per_sec": 5.6487879785017, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 19231, + "tokens_per_sec": 5.45993448078623, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16843, + "tokens_per_sec": 5.937184587068812, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110870, + "tokens_per_sec": 5.276449896274916, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 110774, + "tokens_per_sec": 5.3712965136223305, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 112101, + "tokens_per_sec": 5.379077795916182, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66119, + "tokens_per_sec": 5.580846655273068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 151424, + "tokens_per_sec": 5.197326711749788, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 152836, + "tokens_per_sec": 5.227825904891517, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 88383, + "tokens_per_sec": 5.555366982338232, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 151338, + "tokens_per_sec": 5.226711070583726, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx131k", @@ -393,7 +1947,182 @@ "tokens_per_sec_mean": 4.79, "tokens_per_sec_p50": 4.8, "tokens_per_sec_p95": 4.93, - "tokens_per_sec_max": 4.93 + "tokens_per_sec_max": 4.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 107527.33, + "duration_ms_p50": 120915.0, + "tokens_per_sec_mean": 4.69 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 86488.89, + "duration_ms_p50": 96577.0, + "tokens_per_sec_mean": 4.82 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34153.25, + "duration_ms_p50": 37396.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 4.79 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 98701.0, + "duration_ms_p50": 100326.5, + "tokens_per_sec_mean": 4.82, + "tokens_per_sec_p50": 4.8 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 142391.25, + "duration_ms_p50": 151248.0, + "tokens_per_sec_mean": 4.8, + "tokens_per_sec_p50": 4.79 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1250.42, + "answer_chars_mean": 714.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 34389, + "tokens_per_sec": 4.449097095001308, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18091, + "tokens_per_sec": 4.698468851915317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 40404, + "tokens_per_sec": 4.875754875754875, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43729, + "tokens_per_sec": 4.9166457042237415, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 120915, + "tokens_per_sec": 4.920812140760038, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 73236, + "tokens_per_sec": 4.765415915669889, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 104076, + "tokens_per_sec": 4.82339828586802, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 96577, + "tokens_per_sec": 4.783747683195792, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 167278, + "tokens_per_sec": 4.698764930235895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145866, + "tokens_per_sec": 4.744080183181824, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 99791, + "tokens_per_sec": 4.93030433606237, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 156630, + "tokens_per_sec": 4.839430505011811, + "error": null + } + ] } ], "n_calls_total": 45, @@ -420,7 +2149,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "ad28cb95-5134-40b5-8514-c8a381f83d87", @@ -436,7 +2166,7 @@ "title": "predator-a3b-ngl-ctx-2d-1 \u2014 qwen3 on predator", "date": "2026-05-05", "started_at": "2026-05-05T07:43:00Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -570,7 +2300,182 @@ "tokens_per_sec_mean": 5.48, "tokens_per_sec_p50": 5.31, "tokens_per_sec_p95": 6.79, - "tokens_per_sec_max": 6.8 + "tokens_per_sec_max": 6.8, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92828.67, + "duration_ms_p50": 110762.0, + "tokens_per_sec_mean": 5.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 83155.0, + "duration_ms_p50": 76258.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18907.5, + "duration_ms_p50": 19953.0, + "tokens_per_sec_mean": 6.04, + "tokens_per_sec_p50": 6.21 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 84750.25, + "duration_ms_p50": 78940.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.32 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 153062.5, + "duration_ms_p50": 153912.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.14 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1193.67, + "answer_chars_mean": 809.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 25689, + "tokens_per_sec": 5.644439254155475, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10035, + "tokens_per_sec": 6.776283009466866, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15585, + "tokens_per_sec": 6.801411613731151, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 24321, + "tokens_per_sec": 4.934007647711853, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110762, + "tokens_per_sec": 5.29062313789928, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 76258, + "tokens_per_sec": 5.101104146450209, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70359, + "tokens_per_sec": 5.344021376085505, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 81622, + "tokens_per_sec": 5.451961480973267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 142035, + "tokens_per_sec": 5.252226563875102, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 150027, + "tokens_per_sec": 5.339038972984863, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 162391, + "tokens_per_sec": 5.024909015893738, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 157797, + "tokens_per_sec": 4.835326400375165, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx96k", @@ -582,7 +2487,182 @@ "tokens_per_sec_mean": 5.79, "tokens_per_sec_p50": 5.75, "tokens_per_sec_p95": 6.47, - "tokens_per_sec_max": 6.7 + "tokens_per_sec_max": 6.7, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 65655.0, + "duration_ms_p50": 45721.0, + "tokens_per_sec_mean": 5.92 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 73856.11, + "duration_ms_p50": 59728.0, + "tokens_per_sec_mean": 5.75 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20742.5, + "duration_ms_p50": 18111.0, + "tokens_per_sec_mean": 5.82, + "tokens_per_sec_p50": 5.83 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66764.5, + "duration_ms_p50": 58541.0, + "tokens_per_sec_mean": 5.84, + "tokens_per_sec_p50": 5.76 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 127910.5, + "duration_ms_p50": 132742.5, + "tokens_per_sec_mean": 5.71, + "tokens_per_sec_p50": 5.72 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1209.5, + "answer_chars_mean": 618.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 13701, + "tokens_per_sec": 5.766002481570688, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33047, + "tokens_per_sec": 4.932369050140709, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15830, + "tokens_per_sec": 6.69614655716993, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20392, + "tokens_per_sec": 5.884660651235779, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 45721, + "tokens_per_sec": 6.277203035804117, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59728, + "tokens_per_sec": 5.575274578087329, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 57354, + "tokens_per_sec": 5.666562053213377, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104255, + "tokens_per_sec": 5.85103831950506, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 137543, + "tokens_per_sec": 5.729117439637059, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 144293, + "tokens_per_sec": 5.516553124545196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 101864, + "tokens_per_sec": 5.703683342495877, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127942, + "tokens_per_sec": 5.893295399477889, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx131k", @@ -594,7 +2674,182 @@ "tokens_per_sec_mean": 5.01, "tokens_per_sec_p50": 5.13, "tokens_per_sec_p95": 5.97, - "tokens_per_sec_max": 6.29 + "tokens_per_sec_max": 6.29, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109523.67, + "duration_ms_p50": 118125.0, + "tokens_per_sec_mean": 4.11 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 92033.67, + "duration_ms_p50": 95486.0, + "tokens_per_sec_mean": 5.31 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 33574.75, + "duration_ms_p50": 31872.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 5.19 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 103544.75, + "duration_ms_p50": 105842.0, + "tokens_per_sec_mean": 5.19, + "tokens_per_sec_p50": 5.05 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 152099.0, + "duration_ms_p50": 150187.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1296.25, + "answer_chars_mean": 873.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 59744, + "tokens_per_sec": 2.259641135511516, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10810, + "tokens_per_sec": 6.290471785383904, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 20002, + "tokens_per_sec": 5.049495050494951, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43743, + "tokens_per_sec": 5.326566536360103, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 118125, + "tokens_per_sec": 4.960846560846561, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 116198, + "tokens_per_sec": 5.146388061756657, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95486, + "tokens_per_sec": 4.932660285277422, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 84370, + "tokens_per_sec": 5.71293113665995, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 150702, + "tokens_per_sec": 5.122692465926132, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 148897, + "tokens_per_sec": 5.131063755481978, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 159125, + "tokens_per_sec": 4.788688138256088, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 149672, + "tokens_per_sec": 5.378427494788604, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx48k", @@ -606,7 +2861,182 @@ "tokens_per_sec_mean": 6.6, "tokens_per_sec_p50": 6.59, "tokens_per_sec_p95": 6.89, - "tokens_per_sec_max": 6.91 + "tokens_per_sec_max": 6.91, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66671.33, + "duration_ms_p50": 58574.0, + "tokens_per_sec_mean": 6.36 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 65162.22, + "duration_ms_p50": 69317.0, + "tokens_per_sec_mean": 6.68 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16306.0, + "duration_ms_p50": 15313.0, + "tokens_per_sec_mean": 6.71, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 75223.75, + "duration_ms_p50": 74910.0, + "tokens_per_sec_mean": 6.61, + "tokens_per_sec_p50": 6.59 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 105088.75, + "duration_ms_p50": 114731.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.48 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1152.67, + "answer_chars_mean": 769.33 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19864, + "tokens_per_sec": 6.192106322996375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14734, + "tokens_per_sec": 6.854893443735578, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15636, + "tokens_per_sec": 6.907137375287798, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14990, + "tokens_per_sec": 6.871247498332221, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 58574, + "tokens_per_sec": 6.572882166148803, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 89924, + "tokens_per_sec": 6.60557804368133, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92501, + "tokens_per_sec": 6.551280526697008, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 59896, + "tokens_per_sec": 6.728329103779885, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 121576, + "tokens_per_sec": 6.317036257156018, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 69317, + "tokens_per_sec": 6.6794581415814305, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 114261, + "tokens_per_sec": 6.485152414209573, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 115201, + "tokens_per_sec": 6.475638232307012, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx64k", @@ -618,7 +3048,182 @@ "tokens_per_sec_mean": 6.41, "tokens_per_sec_p50": 6.39, "tokens_per_sec_p95": 6.88, - "tokens_per_sec_max": 6.9 + "tokens_per_sec_max": 6.9, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 80477.67, + "duration_ms_p50": 99471.0, + "tokens_per_sec_mean": 5.99 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 68197.56, + "duration_ms_p50": 69604.0, + "tokens_per_sec_mean": 6.55 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16514.0, + "duration_ms_p50": 16585.0, + "tokens_per_sec_mean": 6.53, + "tokens_per_sec_p50": 6.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 79436.25, + "duration_ms_p50": 81054.0, + "tokens_per_sec_mean": 6.43, + "tokens_per_sec_p50": 6.44 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 117852.5, + "duration_ms_p50": 124494.0, + "tokens_per_sec_mean": 6.25, + "tokens_per_sec_p50": 6.25 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1063.5, + "answer_chars_mean": 944.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 18535, + "tokens_per_sec": 5.557054221742649, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18478, + "tokens_per_sec": 6.873038207598224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 14692, + "tokens_per_sec": 6.806425265450585, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14351, + "tokens_per_sec": 6.898473973939098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 99471, + "tokens_per_sec": 6.2430256054528455, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 56166, + "tokens_per_sec": 6.445180358223837, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92504, + "tokens_per_sec": 6.442964628556602, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 69604, + "tokens_per_sec": 6.5944485949083385, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 123427, + "tokens_per_sec": 6.165587756325601, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 90927, + "tokens_per_sec": 6.345749887272207, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 131495, + "tokens_per_sec": 6.319631925168257, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 125561, + "tokens_per_sec": 6.188227236164095, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx96k", @@ -630,7 +3235,182 @@ "tokens_per_sec_mean": 6.08, "tokens_per_sec_p50": 6.44, "tokens_per_sec_p95": 6.68, - "tokens_per_sec_max": 6.69 + "tokens_per_sec_max": 6.69, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 78350.33, + "duration_ms_p50": 72290.0, + "tokens_per_sec_mean": 4.93 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 66282.56, + "duration_ms_p50": 64420.0, + "tokens_per_sec_mean": 6.47 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25171.0, + "duration_ms_p50": 25926.5, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 6.3 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 73270.5, + "duration_ms_p50": 68355.0, + "tokens_per_sec_mean": 6.39, + "tokens_per_sec_p50": 6.46 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109457.0, + "duration_ms_p50": 117967.0, + "tokens_per_sec_mean": 6.37, + "tokens_per_sec_p50": 6.4 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1185.42, + "answer_chars_mean": 741.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35465, + "tokens_per_sec": 2.7068941209643307, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 27111, + "tokens_per_sec": 6.1598613109070115, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 24742, + "tokens_per_sec": 6.668822245574327, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13366, + "tokens_per_sec": 6.43423612150232, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 72290, + "tokens_per_sec": 5.97593028081339, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92055, + "tokens_per_sec": 6.474390310140676, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 64317, + "tokens_per_sec": 6.436867391202949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 64420, + "tokens_per_sec": 6.690468798509779, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 127296, + "tokens_per_sec": 6.1038838612368025, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 123105, + "tokens_per_sec": 6.344177734454328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 74598, + "tokens_per_sec": 6.568540711547227, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 112829, + "tokens_per_sec": 6.45224188816705, + "error": null + } + ] } ], "n_calls_total": 90, @@ -660,7 +3440,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "79afe366-1055-4e45-adf6-593864a530e8", @@ -676,7 +3457,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T07:00:10Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -798,7 +3579,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -810,7 +3595,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -822,7 +3611,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -834,7 +3627,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:phi-4-q4km-cpu-ctx32k", @@ -846,7 +3643,94 @@ "tokens_per_sec_mean": 0.02, "tokens_per_sec_p50": 0.02, "tokens_per_sec_p95": 0.02, - "tokens_per_sec_max": 0.02 + "tokens_per_sec_max": 0.02, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": 540002.0, + "duration_ms_p50": 540002.0, + "tokens_per_sec_mean": 0.02 + }, + "warm": { + "n_calls": 3, + "duration_ms_mean": 551918.67, + "duration_ms_p50": 515603.0, + "tokens_per_sec_mean": 0.02 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 548939.5, + "duration_ms_p50": 527802.5, + "tokens_per_sec_mean": 0.02, + "tokens_per_sec_p50": 0.02 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 34.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 540002, + "tokens_per_sec": 0.01851844993166692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 628077, + "tokens_per_sec": 0.015921614706477072, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 512076, + "tokens_per_sec": 0.01952835126035979, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 515603, + "tokens_per_sec": 0.019394766903993967, + "error": null + } + ] } ], "n_calls_total": 6, @@ -875,7 +3759,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "212d6278-1b9b-45e9-8aae-7eed4d4ec822", @@ -891,7 +3776,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T03:07:15Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -918,7 +3803,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-cpu-matrix-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -997,7 +3882,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -1009,7 +3898,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -1021,7 +3914,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -1033,7 +3930,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -1059,8 +3960,10 @@ "metadata": "/data/86241185-24da-4f8e-b254-a60c9465e353/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/86241185-24da-4f8e-b254-a60c9465e353/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "212d6278-1b9b-45e9-8aae-7eed4d4ec822", "91751afd-068a-477b-8f40-6e1963f803f1", @@ -1075,7 +3978,7 @@ "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-05", "started_at": "2026-05-05T03:07:13Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -1094,7 +3997,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-gemma-e4b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -1125,7 +4028,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -1148,8 +4055,10 @@ "metadata": "/data/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", "212d6278-1b9b-45e9-8aae-7eed4d4ec822", @@ -1164,7 +4073,7 @@ "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-04", "started_at": "2026-05-04T23:53:09Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -1183,7 +4092,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-gemma-e4b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -1214,7 +4123,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -1237,8 +4150,10 @@ "metadata": "/data/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", "212d6278-1b9b-45e9-8aae-7eed4d4ec822", @@ -1253,7 +4168,7 @@ "title": "predator-a3b-ctx-sweep-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T23:48:01Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -1339,7 +4254,182 @@ "tokens_per_sec_mean": 4.69, "tokens_per_sec_p50": 4.74, "tokens_per_sec_p95": 4.92, - "tokens_per_sec_max": 4.95 + "tokens_per_sec_max": 4.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 90812.0, + "duration_ms_p50": 69978.0, + "tokens_per_sec_mean": 4.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 79562.67, + "duration_ms_p50": 68607.0, + "tokens_per_sec_mean": 4.79 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25406.0, + "duration_ms_p50": 24753.0, + "tokens_per_sec_mean": 4.61, + "tokens_per_sec_p50": 4.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 70149.25, + "duration_ms_p50": 69292.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 151569.75, + "duration_ms_p50": 167047.0, + "tokens_per_sec_mean": 4.7, + "tokens_per_sec_p50": 4.66 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1116.17, + "answer_chars_mean": 630.5 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 33949, + "tokens_per_sec": 3.7998173731185014, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 21340, + "tokens_per_sec": 4.779756326148079, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 28166, + "tokens_per_sec": 4.899524249094653, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18169, + "tokens_per_sec": 4.953492212009467, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 69978, + "tokens_per_sec": 4.730058018234303, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 75274, + "tokens_per_sec": 4.742673433057895, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 68607, + "tokens_per_sec": 4.8828836707624586, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66738, + "tokens_per_sec": 4.689981719560071, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 168509, + "tokens_per_sec": 4.664439288109241, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 173316, + "tokens_per_sec": 4.644695238754645, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98869, + "tokens_per_sec": 4.834680233440209, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 165585, + "tokens_per_sec": 4.662258054775493, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx64k", @@ -1351,7 +4441,182 @@ "tokens_per_sec_mean": 5.46, "tokens_per_sec_p50": 5.41, "tokens_per_sec_p95": 5.73, - "tokens_per_sec_max": 5.83 + "tokens_per_sec_max": 5.83, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 68214.0, + "duration_ms_p50": 89809.0, + "tokens_per_sec_mean": 5.3 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 82623.44, + "duration_ms_p50": 77959.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 26140.75, + "duration_ms_p50": 24427.0, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.61 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 80091.75, + "duration_ms_p50": 85924.0, + "tokens_per_sec_mean": 5.5, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 130830.75, + "duration_ms_p50": 142584.5, + "tokens_per_sec_mean": 5.28, + "tokens_per_sec_p50": 5.29 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1055.92, + "answer_chars_mean": 843.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19560, + "tokens_per_sec": 5.316973415132924, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 29294, + "tokens_per_sec": 5.564279374615962, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 37869, + "tokens_per_sec": 5.651060233964457, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17840, + "tokens_per_sec": 5.829596412556054, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 95273, + "tokens_per_sec": 5.374030417851857, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 93889, + "tokens_per_sec": 5.453247984321912, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 77959, + "tokens_per_sec": 5.579856078194949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 53246, + "tokens_per_sec": 5.577883784697442, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 89809, + "tokens_per_sec": 5.199924283757753, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145855, + "tokens_per_sec": 5.3340646532515175, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 148345, + "tokens_per_sec": 5.244531328996596, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 139314, + "tokens_per_sec": 5.347631968072125, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx128k", @@ -1363,7 +4628,182 @@ "tokens_per_sec_mean": 4.79, "tokens_per_sec_p50": 4.77, "tokens_per_sec_p95": 5.0, - "tokens_per_sec_max": 5.12 + "tokens_per_sec_max": 5.12, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 94769.67, + "duration_ms_p50": 97143.0, + "tokens_per_sec_mean": 4.64 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 87045.33, + "duration_ms_p50": 98125.0, + "tokens_per_sec_mean": 4.84 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25067.0, + "duration_ms_p50": 21998.5, + "tokens_per_sec_mean": 4.72, + "tokens_per_sec_p50": 4.76 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 101212.25, + "duration_ms_p50": 100751.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 140650.0, + "duration_ms_p50": 149948.0, + "tokens_per_sec_mean": 4.88, + "tokens_per_sec_p50": 4.83 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1302.08, + "answer_chars_mean": 611.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 22587, + "tokens_per_sec": 4.471598707220968, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 42874, + "tokens_per_sec": 4.898073424453048, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 21410, + "tokens_per_sec": 4.810836057916862, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13397, + "tokens_per_sec": 4.702545345972979, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97143, + "tokens_per_sec": 4.66322843642877, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 107841, + "tokens_per_sec": 4.747730455021745, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95505, + "tokens_per_sec": 4.732736505942097, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104360, + "tokens_per_sec": 4.906094288999617, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 164579, + "tokens_per_sec": 4.794050273728725, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 161591, + "tokens_per_sec": 4.746551478733345, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98125, + "tokens_per_sec": 5.115923566878981, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 138305, + "tokens_per_sec": 4.858826506633888, + "error": null + } + ] } ], "n_calls_total": 45, @@ -1390,7 +4830,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "79afe366-1055-4e45-adf6-593864a530e8", @@ -1406,7 +4847,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-04", "started_at": "2026-05-04T23:39:53Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -1512,7 +4953,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -1524,7 +4969,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -1536,7 +4985,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -1548,7 +5001,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -1576,7 +5033,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "86241185-24da-4f8e-b254-a60c9465e353", @@ -1592,7 +5050,7 @@ "title": "predator-a3b-ngl-matrix-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T22:50:47Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -1678,7 +5136,174 @@ "tokens_per_sec_mean": 4.95, "tokens_per_sec_p50": 4.98, "tokens_per_sec_p95": 5.15, - "tokens_per_sec_max": 5.15 + "tokens_per_sec_max": 5.15, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 95785.67, + "duration_ms_p50": 108039.0, + "tokens_per_sec_mean": 4.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81986.67, + "duration_ms_p50": 92416.0, + "tokens_per_sec_mean": 4.99 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 24673.5, + "duration_ms_p50": 23088.0, + "tokens_per_sec_mean": 4.84, + "tokens_per_sec_p50": 4.75 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 93075.75, + "duration_ms_p50": 96497.0, + "tokens_per_sec_mean": 4.9, + "tokens_per_sec_p50": 4.88 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 138560.0, + "duration_ms_p50": 153138.0, + "tokens_per_sec_mean": 5.12, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1263.58, + "answer_chars_mean": 656.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 23904, + "tokens_per_sec": 4.769076305220883, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 19868, + "tokens_per_sec": 4.731226092208577, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 22272, + "tokens_per_sec": 4.714439655172414, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 32650, + "tokens_per_sec": 5.145482388973966, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 108039, + "tokens_per_sec": 4.73902942455965, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 71270, + "tokens_per_sec": 4.910902202890417, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92416, + "tokens_per_sec": 4.858466066481995, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 100578, + "tokens_per_sec": 5.0905764680148735, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 155414, + "tokens_per_sec": 5.057459430939298, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92550, + "tokens_per_sec": 5.143165856293895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 153066, + "tokens_per_sec": 5.121973527759267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 153210, + "tokens_per_sec": 5.149794399843352, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl12", @@ -1690,7 +5315,174 @@ "tokens_per_sec_mean": 5.59, "tokens_per_sec_p50": 5.36, "tokens_per_sec_p95": 6.61, - "tokens_per_sec_max": 6.63 + "tokens_per_sec_max": 6.63, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 85208.33, + "duration_ms_p50": 64718.0, + "tokens_per_sec_mean": 5.44 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 71615.33, + "duration_ms_p50": 92482.0, + "tokens_per_sec_mean": 5.64 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20134.0, + "duration_ms_p50": 14613.0, + "tokens_per_sec_mean": 6.15, + "tokens_per_sec_p50": 6.26 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 81221.25, + "duration_ms_p50": 82069.0, + "tokens_per_sec_mean": 5.31, + "tokens_per_sec_p50": 5.31 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 123685.5, + "duration_ms_p50": 126628.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.3 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1129.25, + "answer_chars_mean": 657.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 41903, + "tokens_per_sec": 5.4411378660239125, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10852, + "tokens_per_sec": 6.634721710283818, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 9407, + "tokens_per_sec": 6.590836611034336, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18374, + "tokens_per_sec": 5.932295635136606, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 64718, + "tokens_per_sec": 5.67075620383819, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 96029, + "tokens_per_sec": 5.331722708765061, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 93926, + "tokens_per_sec": 4.971999233439091, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70212, + "tokens_per_sec": 5.28399703754344, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 149004, + "tokens_per_sec": 5.194491423049046, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92482, + "tokens_per_sec": 5.319954153240631, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 108851, + "tokens_per_sec": 5.282450321999798, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 144405, + "tokens_per_sec": 5.387625082233995, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl24", @@ -1702,7 +5494,174 @@ "tokens_per_sec_mean": 6.58, "tokens_per_sec_p50": 6.68, "tokens_per_sec_p95": 6.88, - "tokens_per_sec_max": 6.89 + "tokens_per_sec_max": 6.89, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66010.0, + "duration_ms_p50": 67078.0, + "tokens_per_sec_mean": 6.23 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 61473.11, + "duration_ms_p50": 67020.0, + "tokens_per_sec_mean": 6.7 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 12523.5, + "duration_ms_p50": 12403.5, + "tokens_per_sec_mean": 6.59, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66233.75, + "duration_ms_p50": 67049.0, + "tokens_per_sec_mean": 6.67, + "tokens_per_sec_p50": 6.69 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109064.75, + "duration_ms_p50": 118280.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.44 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1095.0, + "answer_chars_mean": 736.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 11689, + "tokens_per_sec": 5.731884677902301, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14951, + "tokens_per_sec": 6.889171292890108, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 10336, + "tokens_per_sec": 6.8691950464396285, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13118, + "tokens_per_sec": 6.860801951517, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 67078, + "tokens_per_sec": 6.57443573153642, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59196, + "tokens_per_sec": 6.689641191972431, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 71641, + "tokens_per_sec": 6.6861154925252295, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 67020, + "tokens_per_sec": 6.714413607878246, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 119263, + "tokens_per_sec": 6.397625416097196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 72039, + "tokens_per_sec": 6.676938880328711, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 117297, + "tokens_per_sec": 6.462228360486628, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127660, + "tokens_per_sec": 6.423311922293593, + "error": null + } + ] } ], "n_calls_total": 42, @@ -1730,7 +5689,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_NGL_RETUNE_2026-05-05.md" + "A3B_NGL_RETUNE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "79afe366-1055-4e45-adf6-593864a530e8", @@ -1743,10 +5703,10 @@ }, { "id": "23066b38-ea9c-4dd3-b2f5-32912a67fce4", - "title": "Predator Qwen rerun", + "title": "A3B cross-machine \u2014 Pavilion side", "date": "2026-05-04", "started_at": "2026-05-04T22:11:43Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -1762,7 +5722,7 @@ "pavilion", "qwen3" ], - "headline": "14 calls across 1 cell(s); ~5.9 tok/s mean; p50 57.8s", + "headline": "Pavilion (GTX 1050) running Qwen3-30B-A3B IQ2-XXS at ~6 tok/s mean, p50 58s. The smaller half of the cross-machine pair.", "janie_blurb_md": null, "janie_blurb_status": "pending", "caveat": null, @@ -1798,7 +5758,174 @@ "tokens_per_sec_mean": 5.88, "tokens_per_sec_p50": 6.67, "tokens_per_sec_p95": 8.17, - "tokens_per_sec_max": 8.19 + "tokens_per_sec_max": 8.19, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 161561.0, + "duration_ms_p50": 132262.0, + "tokens_per_sec_mean": 3.22 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 42732.89, + "duration_ms_p50": 49471.0, + "tokens_per_sec_mean": 6.77 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 67896.0, + "duration_ms_p50": 16463.0, + "tokens_per_sec_mean": 3.36, + "tokens_per_sec_p50": 3.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 72019.0, + "duration_ms_p50": 55635.5, + "tokens_per_sec_mean": 6.81, + "tokens_per_sec_p50": 7.79 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 77404.75, + "duration_ms_p50": 63543.0, + "tokens_per_sec_mean": 7.47, + "tokens_per_sec_p50": 7.98 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 974.67, + "answer_chars_mean": 587.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 226760, + "tokens_per_sec": 0.28223672605397776, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15935, + "tokens_per_sec": 4.016316284907436, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 16991, + "tokens_per_sec": 3.7667000176564063, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 11898, + "tokens_per_sec": 5.379055303412338, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 132262, + "tokens_per_sec": 3.4855060410397543, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 61800, + "tokens_per_sec": 7.427184466019418, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 49471, + "tokens_per_sec": 8.186614380141902, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 44543, + "tokens_per_sec": 8.149428641986395, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 125661, + "tokens_per_sec": 5.904775546907951, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 68390, + "tokens_per_sec": 7.998245357508408, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 56872, + "tokens_per_sec": 8.018005345336897, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58696, + "tokens_per_sec": 7.956249148153196, + "error": null + } + ] } ], "n_calls_total": 14, @@ -1824,7 +5951,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", @@ -1833,14 +5961,16 @@ "5fb2913d-6500-4ecf-9e97-d43f7dd61145" ], "status": "complete", - "visibility": "draft" + "visibility": "draft", + "headline_override": "Pavilion (GTX 1050) running Qwen3-30B-A3B IQ2-XXS at ~6 tok/s mean, p50 58s. The smaller half of the cross-machine pair.", + "title_override": "A3B cross-machine \u2014 Pavilion side" }, { "id": "d1cff064-9141-48e0-b1d2-729b134e0543", "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T22:02:28Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -1861,7 +5991,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -1892,7 +6022,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -1915,8 +6049,10 @@ "metadata": "/data/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/d1cff064-9141-48e0-b1d2-729b134e0543/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "23066b38-ea9c-4dd3-b2f5-32912a67fce4", "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", @@ -1928,10 +6064,10 @@ }, { "id": "5fb2913d-6500-4ecf-9e97-d43f7dd61145", - "title": "Predator Qwen rerun", + "title": "A3B cross-machine \u2014 Predator side", "date": "2026-05-04", "started_at": "2026-05-04T22:02:17Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -1947,7 +6083,7 @@ "predator", "qwen3" ], - "headline": "14 calls across 1 cell(s); ~3.9 tok/s mean; p50 101.9s", + "headline": "Predator (GTX 1070) running the same A3B IQ2-XXS at ~3.9 tok/s mean, p50 102s \u2014 the larger half of the pair.", "janie_blurb_md": null, "janie_blurb_status": "pending", "caveat": null, @@ -1983,7 +6119,174 @@ "tokens_per_sec_mean": 3.87, "tokens_per_sec_p50": 3.99, "tokens_per_sec_p95": 4.06, - "tokens_per_sec_max": 4.06 + "tokens_per_sec_max": 4.06, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109544.33, + "duration_ms_p50": 97117.0, + "tokens_per_sec_mean": 3.47 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 106568.0, + "duration_ms_p50": 106723.0, + "tokens_per_sec_mean": 4.01 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18483.25, + "duration_ms_p50": 16067.0, + "tokens_per_sec_mean": 3.61, + "tokens_per_sec_p50": 3.95 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 104743.5, + "duration_ms_p50": 101920.0, + "tokens_per_sec_mean": 4.02, + "tokens_per_sec_p50": 4.02 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 198709.5, + "duration_ms_p50": 196861.0, + "tokens_per_sec_mean": 3.99, + "tokens_per_sec_p50": 3.99 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1081.42, + "answer_chars_mean": 835.08 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 26022, + "tokens_per_sec": 2.459457382215049, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15870, + "tokens_per_sec": 3.9697542533081287, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15777, + "tokens_per_sec": 4.056537998352032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16264, + "tokens_per_sec": 3.935071323167733, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97117, + "tokens_per_sec": 3.995181070255465, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 86427, + "tokens_per_sec": 4.061230865354577, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 106723, + "tokens_per_sec": 4.047862222763603, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 128707, + "tokens_per_sec": 3.9780276131057364, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 205494, + "tokens_per_sec": 3.9514535704205476, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 188228, + "tokens_per_sec": 4.037656459187794, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 218750, + "tokens_per_sec": 4.027428571428572, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 182366, + "tokens_per_sec": 3.9590713181185087, + "error": null + } + ] } ], "n_calls_total": 14, @@ -2010,7 +6313,8 @@ "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "A3B_CROSS_MACHINE_2026-05-05.md", - "A3B_NGL_RETUNE_2026-05-05.md" + "A3B_NGL_RETUNE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", @@ -2019,14 +6323,16 @@ "ad28cb95-5134-40b5-8514-c8a381f83d87" ], "status": "complete", - "visibility": "draft" + "visibility": "draft", + "headline_override": "Predator (GTX 1070) running the same A3B IQ2-XXS at ~3.9 tok/s mean, p50 102s \u2014 the larger half of the pair.", + "title_override": "A3B cross-machine \u2014 Predator side" }, { "id": "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T21:24:11Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -2047,7 +6353,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#predator-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -2078,7 +6384,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -2101,8 +6411,10 @@ "metadata": "/data/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "5fb2913d-6500-4ecf-9e97-d43f7dd61145", "fe4fa289-67a9-48e8-8a80-d5d88c875208", @@ -2117,7 +6429,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T21:24:10Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -2138,7 +6450,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -2169,7 +6481,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -2192,8 +6508,10 @@ "metadata": "/data/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "23066b38-ea9c-4dd3-b2f5-32912a67fce4", "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", @@ -2208,7 +6526,7 @@ "title": "pavilion-a3b-1 \u2014 qwen3 on pavilion", "date": "2026-05-04", "started_at": "2026-05-04T20:38:52Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -2229,7 +6547,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -2260,7 +6578,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -2283,8 +6605,10 @@ "metadata": "/data/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "23066b38-ea9c-4dd3-b2f5-32912a67fce4", "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", @@ -2299,7 +6623,7 @@ "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T18:50:19Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -2319,7 +6643,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#predator-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -2350,7 +6674,37 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 2, @@ -2364,8 +6718,8 @@ ], "data_url": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.jsonl" }, - "site_grade": "archive-only", - "site_grade_reason": "no documented method (no synthesis doc and no run.md)", + "site_grade": "standard", + "site_grade_reason": null, "raw_data_urls": { "jsonl": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.jsonl", "log": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.log", @@ -2373,8 +6727,10 @@ "metadata": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [ "5fb2913d-6500-4ecf-9e97-d43f7dd61145", "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", @@ -2386,10 +6742,10 @@ }, { "id": "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", - "title": "Predator Qwen rerun", + "title": "Predator Qwen rerun matrix", "date": "2026-05-04", "started_at": "2026-05-04T17:49:07Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -2406,7 +6762,7 @@ "qwen3.5", "rerun" ], - "headline": "36 calls across 3 cell(s); ~10.9 tok/s mean; p50 24.1s", + "headline": "Three Qwen variants on Predator: 36 calls, ~10.9 tok/s mean, p50 24s. The single-machine flagship for the rerun campaign.", "janie_blurb_md": null, "janie_blurb_status": "pending", "caveat": null, @@ -2474,7 +6830,174 @@ "tokens_per_sec_mean": 14.25, "tokens_per_sec_p50": 14.91, "tokens_per_sec_p95": 15.3, - "tokens_per_sec_max": 15.32 + "tokens_per_sec_max": 15.32, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 32096.67, + "duration_ms_p50": 35949.0, + "tokens_per_sec_mean": 12.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 29933.56, + "duration_ms_p50": 33586.0, + "tokens_per_sec_mean": 14.72 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 5178.25, + "duration_ms_p50": 4671.5, + "tokens_per_sec_mean": 12.69, + "tokens_per_sec_p50": 13.7 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34217.0, + "duration_ms_p50": 33717.5, + "tokens_per_sec_mean": 14.98, + "tokens_per_sec_p50": 15.19 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 52027.75, + "duration_ms_p50": 51815.5, + "tokens_per_sec_mean": 15.09, + "tokens_per_sec_p50": 15.07 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1456.67, + "answer_chars_mean": 434.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6773, + "tokens_per_sec": 9.449283921452828, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4597, + "tokens_per_sec": 13.922123123776375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4632, + "tokens_per_sec": 13.816925734024181, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4711, + "tokens_per_sec": 13.585226066652515, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35949, + "tokens_per_sec": 14.242398954073828, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33849, + "tokens_per_sec": 15.126000768117228, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 33586, + "tokens_per_sec": 15.244447091049842, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 33484, + "tokens_per_sec": 15.290885198900966, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 53568, + "tokens_per_sec": 14.878285543608126, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 51802, + "tokens_per_sec": 14.941508049882245, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 51829, + "tokens_per_sec": 15.203843408130583, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 50912, + "tokens_per_sec": 15.320553111250787, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km-nothink", @@ -2486,7 +7009,174 @@ "tokens_per_sec_mean": 12.55, "tokens_per_sec_p50": 14.37, "tokens_per_sec_p95": 14.93, - "tokens_per_sec_max": 14.95 + "tokens_per_sec_max": 14.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 10193.33, + "duration_ms_p50": 7338.0, + "tokens_per_sec_mean": 11.56 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 10503.22, + "duration_ms_p50": 7062.0, + "tokens_per_sec_mean": 12.89 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1597.75, + "duration_ms_p50": 1393.0, + "tokens_per_sec_mean": 8.47, + "tokens_per_sec_p50": 9.34 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 7203.75, + "duration_ms_p50": 7200.0, + "tokens_per_sec_mean": 14.32, + "tokens_per_sec_p50": 14.37 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22475.75, + "duration_ms_p50": 22106.5, + "tokens_per_sec_mean": 14.87, + "tokens_per_sec_p50": 14.88 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 678.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 2372, + "tokens_per_sec": 5.480607082630692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 1350, + "tokens_per_sec": 9.62962962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 1436, + "tokens_per_sec": 9.052924791086351, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 1233, + "tokens_per_sec": 9.7323600973236, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 7338, + "tokens_per_sec": 14.445352957209048, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 8034, + "tokens_per_sec": 14.438635797859098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 7062, + "tokens_per_sec": 14.301897479467572, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6381, + "tokens_per_sec": 14.104372355430183, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20870, + "tokens_per_sec": 14.758025874460948, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 24820, + "tokens_per_sec": 14.947622884770347, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 23327, + "tokens_per_sec": 14.918334976636514, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20886, + "tokens_per_sec": 14.842478215072298, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:14b-q4km", @@ -2498,7 +7188,118 @@ "tokens_per_sec_mean": 1.06, "tokens_per_sec_p50": 1.09, "tokens_per_sec_p95": 1.28, - "tokens_per_sec_max": 1.33 + "tokens_per_sec_max": 1.33, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": 198723.5, + "duration_ms_p50": 198723.5, + "tokens_per_sec_mean": 0.9 + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": 96952.5, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 1.14 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 68859.25, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 0.97, + "tokens_per_sec_p50": 1.08 + }, + "P-MEDIUM": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": 254910.0, + "duration_ms_p50": 254910.0, + "tokens_per_sec_mean": 1.24, + "tokens_per_sec_p50": 1.24 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 504.5, + "answer_chars_mean": 201.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 98037, + "tokens_per_sec": 0.6528147536134317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59855, + "tokens_per_sec": 1.0692506891654834, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 58654, + "tokens_per_sec": 1.0911446789647765, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58891, + "tokens_per_sec": 1.0867534937426773, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 299410, + "tokens_per_sec": 1.142246417955312, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 210410, + "tokens_per_sec": 1.3307352312152465, + "error": null + } + ] } ], "n_calls_total": 36, @@ -2526,7 +7327,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "09d8fbde-0008-49bb-99da-03eeaca72be1", @@ -2535,14 +7337,16 @@ "fe4fa289-67a9-48e8-8a80-d5d88c875208" ], "status": "complete", - "visibility": "draft" + "visibility": "draft", + "headline_override": "Three Qwen variants on Predator: 36 calls, ~10.9 tok/s mean, p50 24s. The single-machine flagship for the rerun campaign.", + "title_override": "Predator Qwen rerun matrix" }, { "id": "09d8fbde-0008-49bb-99da-03eeaca72be1", "title": "Predator trio bench", "date": "2026-05-04", "started_at": "2026-05-04T16:01:52Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -2630,7 +7434,170 @@ "tokens_per_sec_mean": 13.49, "tokens_per_sec_p50": 15.16, "tokens_per_sec_p95": 15.73, - "tokens_per_sec_max": 15.75 + "tokens_per_sec_max": 15.75, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 9384.0, + "duration_ms_p50": 6095.0, + "tokens_per_sec_mean": 11.89 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 8647.11, + "duration_ms_p50": 6602.0, + "tokens_per_sec_mean": 14.03 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1121.5, + "duration_ms_p50": 918.0, + "tokens_per_sec_mean": 9.73, + "tokens_per_sec_p50": 10.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 6472.5, + "duration_ms_p50": 6403.0, + "tokens_per_sec_mean": 15.05, + "tokens_per_sec_p50": 15.16 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18900.0, + "duration_ms_p50": 18835.5, + "tokens_per_sec_mean": 15.7, + "tokens_per_sec_p50": 15.72 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 1782, + "tokens_per_sec": 5.611672278338945, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 919, + "tokens_per_sec": 10.88139281828074, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 917, + "tokens_per_sec": 10.905125408942203, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 868, + "tokens_per_sec": 11.52073732718894, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6095, + "tokens_per_sec": 14.438063986874488, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 6204, + "tokens_per_sec": 15.151515151515152, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 6989, + "tokens_per_sec": 15.16669051366433, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6602, + "tokens_per_sec": 15.44986367767343, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20275, + "tokens_per_sec": 15.635018495684342, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18761, + "tokens_per_sec": 15.72410852299984, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 18910, + "tokens_per_sec": 15.70597567424643, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17654, + "tokens_per_sec": 15.747139458479666, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:gemma-4:e4b-it-q4km", @@ -2642,7 +7609,170 @@ "tokens_per_sec_mean": 21.82, "tokens_per_sec_p50": 22.91, "tokens_per_sec_p95": 23.64, - "tokens_per_sec_max": 23.66 + "tokens_per_sec_max": 23.66, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 11939.0, + "duration_ms_p50": 3858.0, + "tokens_per_sec_mean": 21.37 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 9204.89, + "duration_ms_p50": 12258.0, + "tokens_per_sec_mean": 21.97 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 2534.5, + "duration_ms_p50": 2884.0, + "tokens_per_sec_mean": 19.17, + "tokens_per_sec_p50": 20.06 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 8322.5, + "duration_ms_p50": 8199.0, + "tokens_per_sec_mean": 22.77, + "tokens_per_sec_p50": 22.85 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18808.25, + "duration_ms_p50": 16034.0, + "tokens_per_sec_mean": 23.52, + "tokens_per_sec_p50": 23.53 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3519, + "tokens_per_sec": 18.186984938903098, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 851, + "tokens_per_sec": 14.10105757931845, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 2918, + "tokens_per_sec": 21.93283070596299, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 2850, + "tokens_per_sec": 22.45614035087719, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3858, + "tokens_per_sec": 22.291342664593053, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4140, + "tokens_per_sec": 21.73913043478261, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 12258, + "tokens_per_sec": 23.413281122532226, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13034, + "tokens_per_sec": 23.63050483351235, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 28440, + "tokens_per_sec": 23.628691983122362, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 16230, + "tokens_per_sec": 23.65988909426987, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15838, + "tokens_per_sec": 23.424674832680896, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14725, + "tokens_per_sec": 23.3616298811545, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km", @@ -2654,7 +7784,170 @@ "tokens_per_sec_mean": 14.04, "tokens_per_sec_p50": 14.42, "tokens_per_sec_p95": 14.57, - "tokens_per_sec_max": 14.57 + "tokens_per_sec_max": 14.57, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 38309.67, + "duration_ms_p50": 35678.0, + "tokens_per_sec_mean": 13.28 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 36745.22, + "duration_ms_p50": 35311.0, + "tokens_per_sec_mean": 14.29 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 4857.5, + "duration_ms_p50": 4646.0, + "tokens_per_sec_mean": 13.25, + "tokens_per_sec_p50": 13.78 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 35390.5, + "duration_ms_p50": 35324.5, + "tokens_per_sec_mean": 14.47, + "tokens_per_sec_p50": 14.49 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71161.0, + "duration_ms_p50": 70320.5, + "tokens_per_sec_mean": 14.4, + "tokens_per_sec_p50": 14.56 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 5514, + "tokens_per_sec": 11.606819006166122, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4659, + "tokens_per_sec": 13.736853402017601, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4633, + "tokens_per_sec": 13.813943449169004, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4624, + "tokens_per_sec": 13.84083044982699, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35678, + "tokens_per_sec": 14.350580189472506, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 35311, + "tokens_per_sec": 14.499730962023166, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 35338, + "tokens_per_sec": 14.488652442130284, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 35235, + "tokens_per_sec": 14.531006101887328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 73737, + "tokens_per_sec": 13.887193674817256, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70291, + "tokens_per_sec": 14.568010129319545, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70266, + "tokens_per_sec": 14.573193294054022, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70350, + "tokens_per_sec": 14.555792466240229, + "error": null + } + ] } ], "n_calls_total": 42, @@ -2682,7 +7975,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [ "5fb2913d-6500-4ecf-9e97-d43f7dd61145", @@ -2693,6 +7987,102 @@ "status": "complete", "visibility": "draft" }, + { + "id": "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T095713Z", + "kind": "legacy", + "title": "HF Chinchilla \u2014 local Mac \u00b7 2026-05-04", + "date": "2026-05-04", + "started_at": "2026-05-04", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "mlx-or-hf", + "harness": "legacy", + "model_family": "chinchilla", + "model_sizes": [ + "hf-chinchilla-mix" + ], + "task_kind": "throughput", + "tags": [ + "legacy", + "mac", + "mac" + ], + "headline": "Legacy archive \u2014 Mac (slobodans-macbook-air) \u00b7 chinchilla", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/mac/hf-chinchilla/hf-chinchilla-local-mac-20260504T095713Z.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/slobodans-macbook-air/reports/hf-chinchilla-benchmarks/hf-chinchilla-local-mac-20260504T095713Z" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "mac", + "legacy_url": "/legacy/mac/hf-chinchilla/hf-chinchilla-local-mac-20260504T095713Z.html", + "legacy_suite": "chinchilla" + }, + { + "id": "legacy:mac:hf-chinchilla:hf-chinchilla-local-mac-20260504T100034Z", + "kind": "legacy", + "title": "HF Chinchilla \u2014 local Mac \u00b7 2026-05-04", + "date": "2026-05-04", + "started_at": "2026-05-04", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "mlx-or-hf", + "harness": "legacy", + "model_family": "chinchilla", + "model_sizes": [ + "hf-chinchilla-mix" + ], + "task_kind": "throughput", + "tags": [ + "legacy", + "mac", + "mac" + ], + "headline": "Legacy archive \u2014 Mac (slobodans-macbook-air) \u00b7 chinchilla", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/mac/hf-chinchilla/hf-chinchilla-local-mac-20260504T100034Z.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/slobodans-macbook-air/reports/hf-chinchilla-benchmarks/hf-chinchilla-local-mac-20260504T100034Z" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "mac", + "legacy_url": "/legacy/mac/hf-chinchilla/hf-chinchilla-local-mac-20260504T100034Z.html", + "legacy_suite": "chinchilla" + }, { "id": "ad057f5b-ed3f-4a95-a38e-361be310ffd6", "title": "pavilion-weeyuga-v3 \u2014 qwen2.5/qwen2.5-coder/qwen3/qwen3.5 on pavilion", @@ -2736,7 +8126,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -3007,7 +8397,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", @@ -3019,7 +8484,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", @@ -3031,7 +8571,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", @@ -3043,7 +8658,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", @@ -3055,7 +8745,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", @@ -3067,7 +8832,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", @@ -3079,7 +8919,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", @@ -3091,7 +9006,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", @@ -3103,7 +9093,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 6, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", @@ -3115,7 +9180,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:14b", @@ -3127,7 +9267,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:8b", @@ -3139,7 +9354,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:4b", @@ -3151,7 +9441,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", @@ -3163,7 +9528,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", @@ -3175,7 +9615,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", @@ -3187,7 +9702,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 96, @@ -3217,7 +9807,7 @@ "data_url": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/run.jsonl" }, "site_grade": "archive-only", - "site_grade_reason": "no documented method (no synthesis doc and no run.md)", + "site_grade_reason": "error rate 12/96", "raw_data_urls": { "jsonl": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/run.jsonl", "log": null, @@ -3225,8 +9815,9 @@ "metadata": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/" }, - "synthesis_doc": "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md", + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md" ], "related_ids": [ @@ -3238,6 +9829,52 @@ "status": "complete", "visibility": "draft" }, + { + "id": "legacy:pavilion:2026-04-29-pavilion-weeyuga-v1-benchmark", + "kind": "legacy", + "title": "Pavilion via Weeyuga \u2014 Mission 1 Phase 1 Baseline", + "date": "2026-04-29", + "started_at": "2026-04-29", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-29-pavilion-weeyuga-v1-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-29-pavilion-weeyuga-v1-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-29-pavilion-weeyuga-v1-benchmark.html", + "legacy_suite": null + }, { "id": "ff1131ca-d021-4e06-8616-4b4cdb54e97e", "title": "pavilion-weeyuga-v1 \u2014 qwen2.5/qwen2.5-coder/qwen3/qwen3.5 on pavilion", @@ -3552,7 +10189,50 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", @@ -3564,7 +10244,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", @@ -3576,7 +10291,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", @@ -3588,7 +10338,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", @@ -3600,7 +10385,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", @@ -3612,7 +10432,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", @@ -3624,7 +10479,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", @@ -3636,7 +10526,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", @@ -3648,7 +10573,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", @@ -3660,7 +10620,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:14b", @@ -3672,7 +10667,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:8b", @@ -3684,7 +10714,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:4b", @@ -3696,7 +10761,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", @@ -3708,7 +10808,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", @@ -3720,7 +10855,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", @@ -3732,7 +10902,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 17, @@ -3773,6 +10978,7 @@ "synthesis_doc": "PAVILION_WEEYUGA_v1.md", "synthesis_docs_all": [ "PAVILION_WEEYUGA_v1.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md" ], "related_ids": [ @@ -3783,6 +10989,4812 @@ ], "status": "complete", "visibility": "draft" + }, + { + "id": "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_12-55-35:report", + "kind": "legacy", + "title": "Prompt Handoff Codegen Benchmark Report", + "date": "2026-04-20", + "started_at": "2026-04-20", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_12-55-35/report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_12-55-35/report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_12-55-35/report.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:prompt-handoff-codegen:prompt-handoff-codegen-2026-04-20_13-08-27:report", + "kind": "legacy", + "title": "Prompt Handoff Codegen Benchmark Report", + "date": "2026-04-20", + "started_at": "2026-04-20", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_13-08-27/report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_13-08-27/report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/prompt-handoff-codegen/prompt-handoff-codegen-2026-04-20_13-08-27/report.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen3_5_0_8b", + "kind": "legacy", + "title": "Qwen3.5 0.8B - Benchmark Page", + "date": "2026-04-18", + "started_at": "2026-04-18", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:0.8b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen3_5_0_8b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen3_5_0_8b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen3_5_0_8b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:report-catalog", + "kind": "legacy", + "title": "Telemetry Report Catalog", + "date": "2026-04-18", + "started_at": "2026-04-18", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/report-catalog.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/report-catalog.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/report-catalog.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-16-qwen3_5_4b-local-mac-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-16-qwen3_5_4b-local-mac-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-16-qwen3_5_4b-local-mac-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:local-mac:qwen3_5_4b_hello_check:2026-04-16-qwen3_5_4b-local-mac-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/qwen3_5_4b_hello_check/2026-04-16-qwen3_5_4b-local-mac-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/qwen3_5_4b_hello_check/2026-04-16-qwen3_5_4b-local-mac-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/qwen3_5_4b_hello_check/2026-04-16-qwen3_5_4b-local-mac-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:models:ahmadwaqar_smolvlm2_256m_video_q8_0", + "kind": "legacy", + "title": "SmolVLM2 256M Video - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/ahmadwaqar_smolvlm2_256m_video_q8_0.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/ahmadwaqar_smolvlm2_256m_video_q8_0.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/ahmadwaqar_smolvlm2_256m_video_q8_0.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:codellama34_16k", + "kind": "legacy", + "title": "CodeLlama 34 16k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "llama", + "model_sizes": [ + "codellama:34b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/codellama34_16k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/codellama34_16k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/codellama34_16k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:codestral_32k", + "kind": "legacy", + "title": "Codestral 32k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "codestral", + "model_sizes": [ + "codestral:22b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/codestral_32k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/codestral_32k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/codestral_32k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:llama3_2_3b", + "kind": "legacy", + "title": "Llama 3.2 3B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "llama", + "model_sizes": [ + "llama3.2:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/llama3_2_3b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/llama3_2_3b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/llama3_2_3b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:phi3", + "kind": "legacy", + "title": "Phi-3 Mini - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "phi", + "model_sizes": [ + "phi-3:mini" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/phi3.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/phi3.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/phi3.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:phind34_16k", + "kind": "legacy", + "title": "Phind 34 16k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "phi", + "model_sizes": [ + "phind-codellama:34b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/phind34_16k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/phind34_16k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/phind34_16k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen14_coder_32k", + "kind": "legacy", + "title": "Qwen14 Coder 32k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen14_coder_32k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen14_coder_32k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen14_coder_32k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen14_general_32k", + "kind": "legacy", + "title": "Qwen14 General 32k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen14_general_32k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen14_general_32k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen14_general_32k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen2_5_3b", + "kind": "legacy", + "title": "Qwen2.5 3B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen2_5_3b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen2_5_3b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen2_5_3b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen2_5_coder_0_5b", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen2_5_coder_0_5b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen2_5_coder_0_5b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen2_5_coder_0_5b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen2_5_coder_14b", + "kind": "legacy", + "title": "Qwen2.5 Coder 14B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:14b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen2_5_coder_14b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen2_5_coder_14b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen2_5_coder_14b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen2_5_coder_1_5b", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen2_5_coder_1_5b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen2_5_coder_1_5b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen2_5_coder_1_5b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen2_5_coder_3b", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen2_5_coder_3b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen2_5_coder_3b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen2_5_coder_3b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen32_coder_32k", + "kind": "legacy", + "title": "Qwen32 Coder 32k - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen32_coder_32k.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen32_coder_32k.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen32_coder_32k.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen3_5_4b", + "kind": "legacy", + "title": "Qwen3.5 4B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen3_5_4b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen3_5_4b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen3_5_4b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:qwen3_5_9b", + "kind": "legacy", + "title": "Qwen3.5 9B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/qwen3_5_9b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/qwen3_5_9b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/qwen3_5_9b.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:riven_smolvlm", + "kind": "legacy", + "title": "SmolVLM 500M - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/riven_smolvlm.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/riven_smolvlm.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/riven_smolvlm.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:smollm2_135m_instruct_q5_k_m", + "kind": "legacy", + "title": "SmolLM2 135M - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "smollm2", + "model_sizes": [ + "smollm2:135m" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/smollm2_135m_instruct_q5_k_m.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/smollm2_135m_instruct_q5_k_m.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/smollm2_135m_instruct_q5_k_m.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:smollm2_1_7b_instruct_q4_k_m", + "kind": "legacy", + "title": "SmolLM2 1.7B - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "smollm2", + "model_sizes": [ + "smollm2:1.7b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/smollm2_1_7b_instruct_q4_k_m.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/smollm2_1_7b_instruct_q4_k_m.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/smollm2_1_7b_instruct_q4_k_m.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:models:smollm2_360m_instruct_q5_k_m", + "kind": "legacy", + "title": "SmolLM2 360M - Benchmark Page", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "smollm2", + "model_sizes": [ + "smollm2:360m" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/models/smollm2_360m_instruct_q5_k_m.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/models/smollm2_360m_instruct_q5_k_m.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/models/smollm2_360m_instruct_q5_k_m.html", + "legacy_suite": null + }, + { + "id": "legacy:mac:2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark", + "kind": "legacy", + "title": "2026-04-16 Qwen3.5 4B MLX Local Mac Benchmark", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b-mlx" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "mac", + "mac" + ], + "headline": "Legacy archive \u2014 Mac (slobodans-macbook-air) \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/mac/2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/slobodans-macbook-air/reports/2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "mac", + "legacy_url": "/legacy/mac/2026-04-16-qwen3_5_4b_mlx-local-mac-benchmark.html", + "legacy_suite": null + }, + { + "id": "legacy:mac:2026-04-16-qwen3_5_4b_mlx-vs-standard-4b", + "kind": "legacy", + "title": "2026-04-16 Qwen3.5 4B MLX vs Qwen3.5 4B Ollama", + "date": "2026-04-16", + "started_at": "2026-04-16", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:4b-mlx" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "mac", + "mac" + ], + "headline": "Legacy archive \u2014 Mac (slobodans-macbook-air) \u00b7 vs-standard", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/mac/2026-04-16-qwen3_5_4b_mlx-vs-standard-4b.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/slobodans-macbook-air/reports/2026-04-16-qwen3_5_4b_mlx-vs-standard-4b.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "mac", + "legacy_url": "/legacy/mac/2026-04-16-qwen3_5_4b_mlx-vs-standard-4b.html", + "legacy_suite": "vs-standard" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-13-qwen3_5_9b-local-mac-progress", + "kind": "legacy", + "title": "Qwen3.5 9B Local Mac Progress", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 progress", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-13-qwen3_5_9b-local-mac-progress.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-13-qwen3_5_9b-local-mac-progress.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-13-qwen3_5_9b-local-mac-progress.html", + "legacy_suite": "progress" + }, + { + "id": "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:qwen3_5_9b:2026-04-13-qwen3_5_9b-vps50-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3_5_9b/2026-04-13-qwen3_5_9b-vps50-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:pavilion:2026-04-13-windows-gpu-qwen35-0_8b-long-context", + "kind": "legacy", + "title": "2026-04-13 Pavilion Windows GPU Qwen3.5 0.8B Long Context", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 long-context", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-0_8b-long-context.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-13-windows-gpu-qwen35-0_8b-long-context.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-0_8b-long-context.html", + "legacy_suite": "long-context" + }, + { + "id": "legacy:pavilion:2026-04-13-windows-gpu-qwen35-2b-long-context", + "kind": "legacy", + "title": "2026-04-13 Pavilion GPU lane - Qwen3.5 2B", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 long-context", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-2b-long-context.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-13-windows-gpu-qwen35-2b-long-context.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-2b-long-context.html", + "legacy_suite": "long-context" + }, + { + "id": "legacy:pavilion:2026-04-13-windows-gpu-qwen35-long-context", + "kind": "legacy", + "title": "2026-04-13 Pavilion Windows GPU Qwen3.5 Long Context", + "date": "2026-04-13", + "started_at": "2026-04-13", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 long-context", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-long-context.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-13-windows-gpu-qwen35-long-context.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-13-windows-gpu-qwen35-long-context.html", + "legacy_suite": "long-context" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model Concurrency Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_0_5b-same-model-concurrency-sweep.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted", + "kind": "legacy", + "title": "Qwen2.5 Coder 14B Local Mac Overview (Aborted)", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:14b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_14b-local-mac-overview-aborted.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model Concurrency Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_1_5b-same-model-concurrency-sweep.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model Concurrency Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-12-qwen2_5_coder_3b-same-model-concurrency-sweep.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:qwen3_5_9b_hello_check:2026-04-12-qwen3_5_9b-local-mac-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen3.5", + "model_sizes": [ + "qwen3.5:9b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/qwen3_5_9b_hello_check/2026-04-12-qwen3_5_9b-local-mac-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/qwen3_5_9b_hello_check/2026-04-12-qwen3_5_9b-local-mac-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/qwen3_5_9b_hello_check/2026-04-12-qwen3_5_9b-local-mac-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-17-36:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 1-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-17-36/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-17-36/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-17-36/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:01-up:qwen2_5_coder_0_5b_1up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 1-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/01-up/qwen2_5_coder_0_5b_1up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:02-up:qwen2_5_coder_0_5b_2up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 2-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/02-up/qwen2_5_coder_0_5b_2up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/02-up/qwen2_5_coder_0_5b_2up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/02-up/qwen2_5_coder_0_5b_2up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:03-up:qwen2_5_coder_0_5b_3up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 3-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/03-up/qwen2_5_coder_0_5b_3up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/03-up/qwen2_5_coder_0_5b_3up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/03-up/qwen2_5_coder_0_5b_3up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:04-up:qwen2_5_coder_0_5b_4up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 4-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/04-up/qwen2_5_coder_0_5b_4up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/04-up/qwen2_5_coder_0_5b_4up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/04-up/qwen2_5_coder_0_5b_4up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:05-up:qwen2_5_coder_0_5b_5up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 5-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/05-up/qwen2_5_coder_0_5b_5up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/05-up/qwen2_5_coder_0_5b_5up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/05-up/qwen2_5_coder_0_5b_5up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:06-up:qwen2_5_coder_0_5b_6up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 6-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/06-up/qwen2_5_coder_0_5b_6up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/06-up/qwen2_5_coder_0_5b_6up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/06-up/qwen2_5_coder_0_5b_6up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_02-18-41:07-up:qwen2_5_coder_0_5b_7up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Same-Model 7-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/07-up/qwen2_5_coder_0_5b_7up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/07-up/qwen2_5_coder_0_5b_7up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_02-18-41/07-up/qwen2_5_coder_0_5b_7up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:03-up:qwen2_5_coder_1_5b_3up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model 3-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/03-up/qwen2_5_coder_1_5b_3up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/03-up/qwen2_5_coder_1_5b_3up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/03-up/qwen2_5_coder_1_5b_3up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:04-up:qwen2_5_coder_1_5b_4up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model 4-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/04-up/qwen2_5_coder_1_5b_4up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/04-up/qwen2_5_coder_1_5b_4up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/04-up/qwen2_5_coder_1_5b_4up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-13-23:05-up:qwen2_5_coder_1_5b_5up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model 5-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/05-up/qwen2_5_coder_1_5b_5up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/05-up/qwen2_5_coder_1_5b_5up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-13-23/05-up/qwen2_5_coder_1_5b_5up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:06-up:qwen2_5_coder_1_5b_6up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model 6-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/06-up/qwen2_5_coder_1_5b_6up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/06-up/qwen2_5_coder_1_5b_6up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/06-up/qwen2_5_coder_1_5b_6up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-22-49:07-up:qwen2_5_coder_1_5b_7up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Same-Model 7-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/07-up/qwen2_5_coder_1_5b_7up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/07-up/qwen2_5_coder_1_5b_7up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-22-49/07-up/qwen2_5_coder_1_5b_7up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:03-up:qwen2_5_coder_3b_3up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model 3-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/03-up/qwen2_5_coder_3b_3up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/03-up/qwen2_5_coder_3b_3up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/03-up/qwen2_5_coder_3b_3up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:04-up:qwen2_5_coder_3b_4up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model 4-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/04-up/qwen2_5_coder_3b_4up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/04-up/qwen2_5_coder_3b_4up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/04-up/qwen2_5_coder_3b_4up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:05-up:qwen2_5_coder_3b_5up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model 5-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/05-up/qwen2_5_coder_3b_5up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/05-up/qwen2_5_coder_3b_5up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/05-up/qwen2_5_coder_3b_5up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:06-up:qwen2_5_coder_3b_6up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model 6-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/06-up/qwen2_5_coder_3b_6up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/06-up/qwen2_5_coder_3b_6up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/06-up/qwen2_5_coder_3b_6up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:vps81-telemetry:local-mac:same_model_concurrency_sweep:same-model-sweep-2026-04-12_17-28-50:07-up:qwen2_5_coder_3b_7up_parallel_report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Same-Model 7-Up Parallel Sweep", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 concurrency", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/07-up/qwen2_5_coder_3b_7up_parallel_report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/07-up/qwen2_5_coder_3b_7up_parallel_report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/same_model_concurrency_sweep/same-model-sweep-2026-04-12_17-28-50/07-up/qwen2_5_coder_3b_7up_parallel_report.html", + "legacy_suite": "concurrency" + }, + { + "id": "legacy:pavilion:2026-04-12-windows-benchmark-coverage-audit", + "kind": "legacy", + "title": "Windows Benchmark Coverage Audit", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 coverage-audit", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-12-windows-benchmark-coverage-audit.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-12-windows-benchmark-coverage-audit.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-12-windows-benchmark-coverage-audit.html", + "legacy_suite": "coverage-audit" + }, + { + "id": "legacy:pavilion:2026-04-12-windows-cpu-qwen14b-benchmark", + "kind": "legacy", + "title": "Qwen2.5 Coder 14B Windows CPU Benchmark", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:14b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-12-windows-cpu-qwen14b-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-12-windows-cpu-qwen14b-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-12-windows-cpu-qwen14b-benchmark.html", + "legacy_suite": null + }, + { + "id": "legacy:pavilion:2026-04-12-windows-gpu-qwen14b-benchmark", + "kind": "legacy", + "title": "Qwen2.5 Coder 14B Windows GPU Benchmark", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:14b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-12-windows-gpu-qwen14b-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-12-windows-gpu-qwen14b-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-12-windows-gpu-qwen14b-benchmark.html", + "legacy_suite": null + }, + { + "id": "legacy:pavilion:_template:windows_benchmark_template", + "kind": "legacy", + "title": "{{DOCUMENT_TITLE}}", + "date": "2026-04-12", + "started_at": "2026-04-12", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/_template/windows_benchmark_template.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/_template/windows_benchmark_template.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/_template/windows_benchmark_template.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-mac-qwen-coder-ladder-overview", + "kind": "legacy", + "title": "Mac Qwen Coder Ladder", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 ladder", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-mac-qwen-coder-ladder-overview.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-mac-qwen-coder-ladder-overview.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-mac-qwen-coder-ladder-overview.html", + "legacy_suite": "ladder" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-mixed-model-20q", + "kind": "legacy", + "title": "Local Mac Qwen Mixed-Model Parallel 20Q", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 parallel", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-parallel-qwen-mixed-model-20q.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-parallel-qwen-mixed-model-20q.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-parallel-qwen-mixed-model-20q.html", + "legacy_suite": "parallel" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-parallel-qwen-same-model-20q", + "kind": "legacy", + "title": "Local Mac Qwen Same-Model Parallel 20Q", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 parallel", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-parallel-qwen-same-model-20q.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-parallel-qwen-same-model-20q.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-parallel-qwen-same-model-20q.html", + "legacy_suite": "parallel" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-small-model-manual", + "kind": "legacy", + "title": "MAC Small-Model Telemetry Manual", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen0_5b-local-mac-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen0_5b-local-mac-vs-vps-report", + "kind": "legacy", + "title": "Qwen0.5B Mac vs VPS Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 mac-vs-vps", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-vs-vps-report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen0_5b-local-mac-vs-vps-report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen0_5b-local-mac-vs-vps-report.html", + "legacy_suite": "mac-vs-vps" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen1_5b-local-mac-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen1_5b-local-mac-vs-vps-report", + "kind": "legacy", + "title": "Qwen2.5 Coder 1.5B Mac vs VPS", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 mac-vs-vps", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-vs-vps-report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen1_5b-local-mac-vs-vps-report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen1_5b-local-mac-vs-vps-report.html", + "legacy_suite": "mac-vs-vps" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen3b-local-mac-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:local-mac:2026-04-11-qwen3b-local-mac-vs-vps-report", + "kind": "legacy", + "title": "Qwen2.5 Coder 3B Mac vs VPS", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 mac-vs-vps", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-vs-vps-report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/2026-04-11-qwen3b-local-mac-vs-vps-report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/2026-04-11-qwen3b-local-mac-vs-vps-report.html", + "legacy_suite": "mac-vs-vps" + }, + { + "id": "legacy:vps81-telemetry:local-mac:qwen0_5b_hello_check:2026-04-11-qwen0_5b-local-mac-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/qwen0_5b_hello_check/2026-04-11-qwen0_5b-local-mac-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/qwen0_5b_hello_check/2026-04-11-qwen0_5b-local-mac-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/qwen0_5b_hello_check/2026-04-11-qwen0_5b-local-mac-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:local-mac:qwen1_5b_hello_check:2026-04-11-qwen1_5b-local-mac-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/qwen1_5b_hello_check/2026-04-11-qwen1_5b-local-mac-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/qwen1_5b_hello_check/2026-04-11-qwen1_5b-local-mac-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/qwen1_5b_hello_check/2026-04-11-qwen1_5b-local-mac-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:local-mac:qwen3b_hello_check:2026-04-11-qwen3b-local-mac-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "mac", + "hardware_label": "Mac \u00b7 M1 MacBook Air \u00b7 8 GB unified \u00b7 8-core", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "mac" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/local-mac/qwen3b_hello_check/2026-04-11-qwen3b-local-mac-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/local-mac/qwen3b_hello_check/2026-04-11-qwen3b-local-mac-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/local-mac/qwen3b_hello_check/2026-04-11-qwen3b-local-mac-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:qwen1_5b:2026-04-11-qwen1_5b-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:1.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen1_5b/2026-04-11-qwen1_5b-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen1_5b/2026-04-11-qwen1_5b-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3b/2026-04-11-qwen3b-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3b/2026-04-11-qwen3b-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:qwen3b:2026-04-11-qwen3b-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5", + "model_sizes": [ + "qwen2.5:3b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen3b/2026-04-11-qwen3b-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen3b/2026-04-11-qwen3b-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:pavilion:2026-04-11-windows-cpu-qwen0_5b-benchmark", + "kind": "legacy", + "title": "Windows Pavilion qwen0.5b CPU Benchmark", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-11-windows-cpu-qwen0_5b-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-11-windows-cpu-qwen0_5b-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-11-windows-cpu-qwen0_5b-benchmark.html", + "legacy_suite": null + }, + { + "id": "legacy:pavilion:2026-04-11-windows-gpu-qwen0_5b-benchmark", + "kind": "legacy", + "title": "Windows Pavilion qwen0.5b GPU Benchmark", + "date": "2026-04-11", + "started_at": "2026-04-11", + "git_sha": null, + "hardware": "pavilion", + "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "pavilion", + "pavilion" + ], + "headline": "Legacy archive \u2014 Pavilion windows-laptop \u00b7 general", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/pavilion/2026-04-11-windows-gpu-qwen0_5b-benchmark.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/pavilion-windows-laptop/reports/2026-04-11-windows-gpu-qwen0_5b-benchmark.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "pavilion", + "legacy_url": "/legacy/pavilion/2026-04-11-windows-gpu-qwen0_5b-benchmark.html", + "legacy_suite": null + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Benchmark Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 benchmark-report", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-initial-5q-20q.html", + "legacy_suite": "benchmark-report" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report-with-10q", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Benchmark Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 benchmark-report", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-with-10q.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-with-10q.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report-with-10q.html", + "legacy_suite": "benchmark-report" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-benchmark-report", + "kind": "legacy", + "title": "Qwen2.5 Coder 0.5B Benchmark Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 benchmark-report", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-benchmark-report.html", + "legacy_suite": "benchmark-report" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-20q-v1", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v1-20q-mini.html", + "legacy_suite": "python-task-20q-v1" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-task-10q-v2", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-python-task-suite-v2-10q-mini.html", + "legacy_suite": "python-task-10q-v2" + }, + { + "id": "legacy:vps81-telemetry:qwen0_5b:2026-04-10-qwen0_5b-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-10", + "started_at": "2026-04-10", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "qwen2.5-coder", + "model_sizes": [ + "qwen2.5-coder:0.5b" + ], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/qwen0_5b/2026-04-10-qwen0_5b-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/qwen0_5b/2026-04-10-qwen0_5b-small-model-manual.html", + "legacy_suite": "small-model-manual" + }, + { + "id": "legacy:vps81-telemetry:python-overnight:2026-04-09-python-task-suite-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-09", + "started_at": "2026-04-09", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-overnight", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/python-overnight/2026-04-09-python-task-suite-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/python-overnight/2026-04-09-python-task-suite-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/python-overnight/2026-04-09-python-task-suite-mini.html", + "legacy_suite": "python-overnight" + }, + { + "id": "legacy:vps81-telemetry:python-overnight:2026-04-06-python-task-suite-mini", + "kind": "legacy", + "title": "vps50 Python Telemetry Mini Report", + "date": "2026-04-06", + "started_at": "2026-04-06", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 python-overnight", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/python-overnight/2026-04-06-python-task-suite-mini.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/python-overnight/2026-04-06-python-task-suite-mini.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/python-overnight/2026-04-06-python-task-suite-mini.html", + "legacy_suite": "python-overnight" + }, + { + "id": "legacy:vps81-telemetry:2026-04-05-large-model-hello-check", + "kind": "legacy", + "title": "Hello Check Report", + "date": "2026-04-05", + "started_at": "2026-04-05", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 hello-check", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/2026-04-05-large-model-hello-check.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/2026-04-05-large-model-hello-check.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/2026-04-05-large-model-hello-check.html", + "legacy_suite": "hello-check" + }, + { + "id": "legacy:vps81-telemetry:2026-04-05-small-model-manual", + "kind": "legacy", + "title": "vps50 Small-Model Telemetry Manual", + "date": "2026-04-05", + "started_at": "2026-04-05", + "git_sha": null, + "hardware": "vps50", + "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", + "engine": "ollama", + "harness": "legacy", + "model_family": "unknown", + "model_sizes": [], + "task_kind": "chat", + "tags": [ + "legacy", + "vps81-telemetry", + "vps50" + ], + "headline": "Legacy archive \u2014 vps-81 historical telemetry \u00b7 small-model-manual", + "janie_blurb_md": null, + "janie_blurb_status": "n-a-legacy", + "caveat": null, + "caveat_severity": null, + "methodology_ref": "legacy", + "methodology_deviations_md": null, + "results_table": [], + "cells": [], + "n_calls_total": 0, + "n_errors_total": 0, + "chart_spec": null, + "site_grade": "legacy", + "site_grade_reason": "historical archive \u2014 methodology may differ", + "raw_data_urls": { + "legacy_html": "/legacy/vps81-telemetry/2026-04-05-small-model-manual.html", + "source_path": "/Users/slobodan/Documents/MyServers/instances/vps-81-17-99-14/telemetry/generated/2026-04-05-small-model-manual.html" + }, + "synthesis_doc": null, + "synthesis_docs_all": [], + "related_ids": [], + "status": "complete", + "visibility": "draft", + "legacy_source_id": "vps81-telemetry", + "legacy_url": "/legacy/vps81-telemetry/2026-04-05-small-model-manual.html", + "legacy_suite": "small-model-manual" } ], "skipped": [ diff --git a/runs/09d8fbde-0008-49bb-99da-03eeaca72be1/metadata.json b/runs/09d8fbde-0008-49bb-99da-03eeaca72be1/metadata.json index dddb2ee..185a650 100644 --- a/runs/09d8fbde-0008-49bb-99da-03eeaca72be1/metadata.json +++ b/runs/09d8fbde-0008-49bb-99da-03eeaca72be1/metadata.json @@ -3,7 +3,7 @@ "title": "Predator trio bench", "date": "2026-05-04", "started_at": "2026-05-04T16:01:52Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -91,7 +91,170 @@ "tokens_per_sec_mean": 13.49, "tokens_per_sec_p50": 15.16, "tokens_per_sec_p95": 15.73, - "tokens_per_sec_max": 15.75 + "tokens_per_sec_max": 15.75, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 9384.0, + "duration_ms_p50": 6095.0, + "tokens_per_sec_mean": 11.89 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 8647.11, + "duration_ms_p50": 6602.0, + "tokens_per_sec_mean": 14.03 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1121.5, + "duration_ms_p50": 918.0, + "tokens_per_sec_mean": 9.73, + "tokens_per_sec_p50": 10.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 6472.5, + "duration_ms_p50": 6403.0, + "tokens_per_sec_mean": 15.05, + "tokens_per_sec_p50": 15.16 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18900.0, + "duration_ms_p50": 18835.5, + "tokens_per_sec_mean": 15.7, + "tokens_per_sec_p50": 15.72 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 1782, + "tokens_per_sec": 5.611672278338945, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 919, + "tokens_per_sec": 10.88139281828074, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 917, + "tokens_per_sec": 10.905125408942203, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 868, + "tokens_per_sec": 11.52073732718894, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6095, + "tokens_per_sec": 14.438063986874488, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 6204, + "tokens_per_sec": 15.151515151515152, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 6989, + "tokens_per_sec": 15.16669051366433, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6602, + "tokens_per_sec": 15.44986367767343, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20275, + "tokens_per_sec": 15.635018495684342, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18761, + "tokens_per_sec": 15.72410852299984, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 18910, + "tokens_per_sec": 15.70597567424643, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17654, + "tokens_per_sec": 15.747139458479666, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:gemma-4:e4b-it-q4km", @@ -103,7 +266,170 @@ "tokens_per_sec_mean": 21.82, "tokens_per_sec_p50": 22.91, "tokens_per_sec_p95": 23.64, - "tokens_per_sec_max": 23.66 + "tokens_per_sec_max": 23.66, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 11939.0, + "duration_ms_p50": 3858.0, + "tokens_per_sec_mean": 21.37 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 9204.89, + "duration_ms_p50": 12258.0, + "tokens_per_sec_mean": 21.97 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 2534.5, + "duration_ms_p50": 2884.0, + "tokens_per_sec_mean": 19.17, + "tokens_per_sec_p50": 20.06 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 8322.5, + "duration_ms_p50": 8199.0, + "tokens_per_sec_mean": 22.77, + "tokens_per_sec_p50": 22.85 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18808.25, + "duration_ms_p50": 16034.0, + "tokens_per_sec_mean": 23.52, + "tokens_per_sec_p50": 23.53 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3519, + "tokens_per_sec": 18.186984938903098, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 851, + "tokens_per_sec": 14.10105757931845, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 2918, + "tokens_per_sec": 21.93283070596299, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 2850, + "tokens_per_sec": 22.45614035087719, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3858, + "tokens_per_sec": 22.291342664593053, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4140, + "tokens_per_sec": 21.73913043478261, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 12258, + "tokens_per_sec": 23.413281122532226, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13034, + "tokens_per_sec": 23.63050483351235, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 28440, + "tokens_per_sec": 23.628691983122362, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 16230, + "tokens_per_sec": 23.65988909426987, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15838, + "tokens_per_sec": 23.424674832680896, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14725, + "tokens_per_sec": 23.3616298811545, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km", @@ -115,7 +441,170 @@ "tokens_per_sec_mean": 14.04, "tokens_per_sec_p50": 14.42, "tokens_per_sec_p95": 14.57, - "tokens_per_sec_max": 14.57 + "tokens_per_sec_max": 14.57, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 38309.67, + "duration_ms_p50": 35678.0, + "tokens_per_sec_mean": 13.28 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 36745.22, + "duration_ms_p50": 35311.0, + "tokens_per_sec_mean": 14.29 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 4857.5, + "duration_ms_p50": 4646.0, + "tokens_per_sec_mean": 13.25, + "tokens_per_sec_p50": 13.78 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 35390.5, + "duration_ms_p50": 35324.5, + "tokens_per_sec_mean": 14.47, + "tokens_per_sec_p50": 14.49 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71161.0, + "duration_ms_p50": 70320.5, + "tokens_per_sec_mean": 14.4, + "tokens_per_sec_p50": 14.56 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 5514, + "tokens_per_sec": 11.606819006166122, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4659, + "tokens_per_sec": 13.736853402017601, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4633, + "tokens_per_sec": 13.813943449169004, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4624, + "tokens_per_sec": 13.84083044982699, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35678, + "tokens_per_sec": 14.350580189472506, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 35311, + "tokens_per_sec": 14.499730962023166, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 35338, + "tokens_per_sec": 14.488652442130284, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 35235, + "tokens_per_sec": 14.531006101887328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 73737, + "tokens_per_sec": 13.887193674817256, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70291, + "tokens_per_sec": 14.568010129319545, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70266, + "tokens_per_sec": 14.573193294054022, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70350, + "tokens_per_sec": 14.555792466240229, + "error": null + } + ] } ], "n_calls_total": 42, @@ -143,7 +632,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -160,6 +650,169 @@ "tokens_per_sec_p50": 15.16, "tokens_per_sec_p95": 15.73, "tokens_per_sec_max": 15.75, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 9384.0, + "duration_ms_p50": 6095.0, + "tokens_per_sec_mean": 11.89 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 8647.11, + "duration_ms_p50": 6602.0, + "tokens_per_sec_mean": 14.03 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1121.5, + "duration_ms_p50": 918.0, + "tokens_per_sec_mean": 9.73, + "tokens_per_sec_p50": 10.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 6472.5, + "duration_ms_p50": 6403.0, + "tokens_per_sec_mean": 15.05, + "tokens_per_sec_p50": 15.16 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18900.0, + "duration_ms_p50": 18835.5, + "tokens_per_sec_mean": 15.7, + "tokens_per_sec_p50": 15.72 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 1782, + "tokens_per_sec": 5.611672278338945, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 919, + "tokens_per_sec": 10.88139281828074, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 917, + "tokens_per_sec": 10.905125408942203, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 868, + "tokens_per_sec": 11.52073732718894, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6095, + "tokens_per_sec": 14.438063986874488, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 6204, + "tokens_per_sec": 15.151515151515152, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 6989, + "tokens_per_sec": 15.16669051366433, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6602, + "tokens_per_sec": 15.44986367767343, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20275, + "tokens_per_sec": 15.635018495684342, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18761, + "tokens_per_sec": 15.72410852299984, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 18910, + "tokens_per_sec": 15.70597567424643, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17654, + "tokens_per_sec": 15.747139458479666, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -354,6 +1007,169 @@ "tokens_per_sec_p50": 22.91, "tokens_per_sec_p95": 23.64, "tokens_per_sec_max": 23.66, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 11939.0, + "duration_ms_p50": 3858.0, + "tokens_per_sec_mean": 21.37 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 9204.89, + "duration_ms_p50": 12258.0, + "tokens_per_sec_mean": 21.97 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 2534.5, + "duration_ms_p50": 2884.0, + "tokens_per_sec_mean": 19.17, + "tokens_per_sec_p50": 20.06 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 8322.5, + "duration_ms_p50": 8199.0, + "tokens_per_sec_mean": 22.77, + "tokens_per_sec_p50": 22.85 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18808.25, + "duration_ms_p50": 16034.0, + "tokens_per_sec_mean": 23.52, + "tokens_per_sec_p50": 23.53 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3519, + "tokens_per_sec": 18.186984938903098, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 851, + "tokens_per_sec": 14.10105757931845, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 2918, + "tokens_per_sec": 21.93283070596299, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 2850, + "tokens_per_sec": 22.45614035087719, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 3858, + "tokens_per_sec": 22.291342664593053, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4140, + "tokens_per_sec": 21.73913043478261, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 12258, + "tokens_per_sec": 23.413281122532226, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13034, + "tokens_per_sec": 23.63050483351235, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 28440, + "tokens_per_sec": 23.628691983122362, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 16230, + "tokens_per_sec": 23.65988909426987, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15838, + "tokens_per_sec": 23.424674832680896, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14725, + "tokens_per_sec": 23.3616298811545, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -548,6 +1364,169 @@ "tokens_per_sec_p50": 14.42, "tokens_per_sec_p95": 14.57, "tokens_per_sec_max": 14.57, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 38309.67, + "duration_ms_p50": 35678.0, + "tokens_per_sec_mean": 13.28 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 36745.22, + "duration_ms_p50": 35311.0, + "tokens_per_sec_mean": 14.29 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 4857.5, + "duration_ms_p50": 4646.0, + "tokens_per_sec_mean": 13.25, + "tokens_per_sec_p50": 13.78 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 35390.5, + "duration_ms_p50": 35324.5, + "tokens_per_sec_mean": 14.47, + "tokens_per_sec_p50": 14.49 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71161.0, + "duration_ms_p50": 70320.5, + "tokens_per_sec_mean": 14.4, + "tokens_per_sec_p50": 14.56 + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 5514, + "tokens_per_sec": 11.606819006166122, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4659, + "tokens_per_sec": 13.736853402017601, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4633, + "tokens_per_sec": 13.813943449169004, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4624, + "tokens_per_sec": 13.84083044982699, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35678, + "tokens_per_sec": 14.350580189472506, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 35311, + "tokens_per_sec": 14.499730962023166, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 35338, + "tokens_per_sec": 14.488652442130284, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 35235, + "tokens_per_sec": 14.531006101887328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 73737, + "tokens_per_sec": 13.887193674817256, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70291, + "tokens_per_sec": 14.568010129319545, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70266, + "tokens_per_sec": 14.573193294054022, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70350, + "tokens_per_sec": 14.555792466240229, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -756,5 +1735,5 @@ "engine": "llamacpp", "predator_target_url": "http://10.8.0.7:11436" }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json b/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json index 76498d4..01941c9 100644 --- a/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json +++ b/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-05", "started_at": "2026-05-05T03:07:13Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -22,7 +22,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-gemma-e4b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -53,7 +53,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -76,8 +80,10 @@ "metadata": "/data/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -93,6 +99,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -112,5 +122,5 @@ "ngl": 0, "ctx": 32768 }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822/metadata.json b/runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822/metadata.json index 60570ca..908d78e 100644 --- a/runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822/metadata.json +++ b/runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-04", "started_at": "2026-05-04T23:39:53Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -109,7 +109,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -121,7 +125,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -133,7 +141,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -145,7 +157,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -173,7 +189,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "meta-only", @@ -190,6 +207,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -203,6 +224,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -216,6 +241,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -229,6 +258,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -255,5 +288,5 @@ "qwen2.5-72b" ] }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4/metadata.json b/runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4/metadata.json index fe6ff13..eff4264 100644 --- a/runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4/metadata.json +++ b/runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4/metadata.json @@ -3,7 +3,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T22:11:43Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -55,7 +55,174 @@ "tokens_per_sec_mean": 5.88, "tokens_per_sec_p50": 6.67, "tokens_per_sec_p95": 8.17, - "tokens_per_sec_max": 8.19 + "tokens_per_sec_max": 8.19, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 161561.0, + "duration_ms_p50": 132262.0, + "tokens_per_sec_mean": 3.22 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 42732.89, + "duration_ms_p50": 49471.0, + "tokens_per_sec_mean": 6.77 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 67896.0, + "duration_ms_p50": 16463.0, + "tokens_per_sec_mean": 3.36, + "tokens_per_sec_p50": 3.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 72019.0, + "duration_ms_p50": 55635.5, + "tokens_per_sec_mean": 6.81, + "tokens_per_sec_p50": 7.79 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 77404.75, + "duration_ms_p50": 63543.0, + "tokens_per_sec_mean": 7.47, + "tokens_per_sec_p50": 7.98 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 974.67, + "answer_chars_mean": 587.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 226760, + "tokens_per_sec": 0.28223672605397776, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15935, + "tokens_per_sec": 4.016316284907436, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 16991, + "tokens_per_sec": 3.7667000176564063, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 11898, + "tokens_per_sec": 5.379055303412338, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 132262, + "tokens_per_sec": 3.4855060410397543, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 61800, + "tokens_per_sec": 7.427184466019418, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 49471, + "tokens_per_sec": 8.186614380141902, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 44543, + "tokens_per_sec": 8.149428641986395, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 125661, + "tokens_per_sec": 5.904775546907951, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 68390, + "tokens_per_sec": 7.998245357508408, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 56872, + "tokens_per_sec": 8.018005345336897, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58696, + "tokens_per_sec": 7.956249148153196, + "error": null + } + ] } ], "n_calls_total": 14, @@ -81,7 +248,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -98,6 +266,173 @@ "tokens_per_sec_p50": 6.67, "tokens_per_sec_p95": 8.17, "tokens_per_sec_max": 8.19, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 161561.0, + "duration_ms_p50": 132262.0, + "tokens_per_sec_mean": 3.22 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 42732.89, + "duration_ms_p50": 49471.0, + "tokens_per_sec_mean": 6.77 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 67896.0, + "duration_ms_p50": 16463.0, + "tokens_per_sec_mean": 3.36, + "tokens_per_sec_p50": 3.89 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 72019.0, + "duration_ms_p50": 55635.5, + "tokens_per_sec_mean": 6.81, + "tokens_per_sec_p50": 7.79 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 77404.75, + "duration_ms_p50": 63543.0, + "tokens_per_sec_mean": 7.47, + "tokens_per_sec_p50": 7.98 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 974.67, + "answer_chars_mean": 587.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 226760, + "tokens_per_sec": 0.28223672605397776, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15935, + "tokens_per_sec": 4.016316284907436, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 16991, + "tokens_per_sec": 3.7667000176564063, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 11898, + "tokens_per_sec": 5.379055303412338, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 132262, + "tokens_per_sec": 3.4855060410397543, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 61800, + "tokens_per_sec": 7.427184466019418, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 49471, + "tokens_per_sec": 8.186614380141902, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 44543, + "tokens_per_sec": 8.149428641986395, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 125661, + "tokens_per_sec": 5.904775546907951, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 68390, + "tokens_per_sec": 7.998245357508408, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 56872, + "tokens_per_sec": 8.018005345336897, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58696, + "tokens_per_sec": 7.956249148153196, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -327,5 +662,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json b/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json index 87117bf..88bbcee 100644 --- a/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json +++ b/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json @@ -3,7 +3,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T21:24:10Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -24,7 +24,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -55,7 +55,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -78,8 +82,10 @@ "metadata": "/data/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -95,6 +101,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -119,5 +129,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json b/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json index b171749..59856eb 100644 --- a/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json +++ b/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json @@ -3,7 +3,7 @@ "title": "pavilion-a3b-1 \u2014 qwen3 on pavilion", "date": "2026-05-04", "started_at": "2026-05-04T20:38:52Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -24,7 +24,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -55,7 +55,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -78,8 +82,10 @@ "metadata": "/data/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -95,6 +101,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -119,5 +129,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145/metadata.json b/runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145/metadata.json index 45402df..edc9250 100644 --- a/runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145/metadata.json +++ b/runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145/metadata.json @@ -3,7 +3,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T22:02:17Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -55,7 +55,174 @@ "tokens_per_sec_mean": 3.87, "tokens_per_sec_p50": 3.99, "tokens_per_sec_p95": 4.06, - "tokens_per_sec_max": 4.06 + "tokens_per_sec_max": 4.06, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109544.33, + "duration_ms_p50": 97117.0, + "tokens_per_sec_mean": 3.47 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 106568.0, + "duration_ms_p50": 106723.0, + "tokens_per_sec_mean": 4.01 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18483.25, + "duration_ms_p50": 16067.0, + "tokens_per_sec_mean": 3.61, + "tokens_per_sec_p50": 3.95 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 104743.5, + "duration_ms_p50": 101920.0, + "tokens_per_sec_mean": 4.02, + "tokens_per_sec_p50": 4.02 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 198709.5, + "duration_ms_p50": 196861.0, + "tokens_per_sec_mean": 3.99, + "tokens_per_sec_p50": 3.99 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1081.42, + "answer_chars_mean": 835.08 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 26022, + "tokens_per_sec": 2.459457382215049, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15870, + "tokens_per_sec": 3.9697542533081287, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15777, + "tokens_per_sec": 4.056537998352032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16264, + "tokens_per_sec": 3.935071323167733, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97117, + "tokens_per_sec": 3.995181070255465, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 86427, + "tokens_per_sec": 4.061230865354577, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 106723, + "tokens_per_sec": 4.047862222763603, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 128707, + "tokens_per_sec": 3.9780276131057364, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 205494, + "tokens_per_sec": 3.9514535704205476, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 188228, + "tokens_per_sec": 4.037656459187794, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 218750, + "tokens_per_sec": 4.027428571428572, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 182366, + "tokens_per_sec": 3.9590713181185087, + "error": null + } + ] } ], "n_calls_total": 14, @@ -82,7 +249,8 @@ "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "A3B_CROSS_MACHINE_2026-05-05.md", - "A3B_NGL_RETUNE_2026-05-05.md" + "A3B_NGL_RETUNE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -99,6 +267,173 @@ "tokens_per_sec_p50": 3.99, "tokens_per_sec_p95": 4.06, "tokens_per_sec_max": 4.06, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109544.33, + "duration_ms_p50": 97117.0, + "tokens_per_sec_mean": 3.47 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 106568.0, + "duration_ms_p50": 106723.0, + "tokens_per_sec_mean": 4.01 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18483.25, + "duration_ms_p50": 16067.0, + "tokens_per_sec_mean": 3.61, + "tokens_per_sec_p50": 3.95 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 104743.5, + "duration_ms_p50": 101920.0, + "tokens_per_sec_mean": 4.02, + "tokens_per_sec_p50": 4.02 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 198709.5, + "duration_ms_p50": 196861.0, + "tokens_per_sec_mean": 3.99, + "tokens_per_sec_p50": 3.99 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1081.42, + "answer_chars_mean": 835.08 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 26022, + "tokens_per_sec": 2.459457382215049, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15870, + "tokens_per_sec": 3.9697542533081287, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15777, + "tokens_per_sec": 4.056537998352032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16264, + "tokens_per_sec": 3.935071323167733, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97117, + "tokens_per_sec": 3.995181070255465, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 86427, + "tokens_per_sec": 4.061230865354577, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 106723, + "tokens_per_sec": 4.047862222763603, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 128707, + "tokens_per_sec": 3.9780276131057364, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 205494, + "tokens_per_sec": 3.9514535704205476, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 188228, + "tokens_per_sec": 4.037656459187794, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 218750, + "tokens_per_sec": 4.027428571428572, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 182366, + "tokens_per_sec": 3.9590713181185087, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -326,5 +661,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/79afe366-1055-4e45-adf6-593864a530e8/metadata.json b/runs/79afe366-1055-4e45-adf6-593864a530e8/metadata.json index e16f2ca..b6c7605 100644 --- a/runs/79afe366-1055-4e45-adf6-593864a530e8/metadata.json +++ b/runs/79afe366-1055-4e45-adf6-593864a530e8/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-ngl6-ctx-1 \u2014 qwen3 on predator", "date": "2026-05-05", "started_at": "2026-05-05T09:44:34Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -89,7 +89,182 @@ "tokens_per_sec_mean": 5.42, "tokens_per_sec_p50": 5.35, "tokens_per_sec_p95": 5.91, - "tokens_per_sec_max": 5.93 + "tokens_per_sec_max": 5.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 88941.0, + "duration_ms_p50": 81140.0, + "tokens_per_sec_mean": 5.21 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 78787.89, + "duration_ms_p50": 70839.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22599.5, + "duration_ms_p50": 19202.0, + "tokens_per_sec_mean": 5.54, + "tokens_per_sec_p50": 5.54 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71981.0, + "duration_ms_p50": 73215.0, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 149398.0, + "duration_ms_p50": 146891.5, + "tokens_per_sec_mean": 5.23, + "tokens_per_sec_p50": 5.26 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1030.83, + "answer_chars_mean": 891.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 38170, + "tokens_per_sec": 5.187319884726224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15973, + "tokens_per_sec": 5.884930820760032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 13824, + "tokens_per_sec": 5.931712962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 22431, + "tokens_per_sec": 5.171414560206856, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 81140, + "tokens_per_sec": 5.385753019472516, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70839, + "tokens_per_sec": 5.533674953062579, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 60354, + "tokens_per_sec": 5.550584882526428, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 75591, + "tokens_per_sec": 5.5033006574856795, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 147513, + "tokens_per_sec": 5.063960464501434, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 157591, + "tokens_per_sec": 5.254107150789068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 146270, + "tokens_per_sec": 5.271074041156765, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 146218, + "tokens_per_sec": 5.313983230518815, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx96k", @@ -101,7 +276,182 @@ "tokens_per_sec_mean": 5.43, "tokens_per_sec_p50": 5.38, "tokens_per_sec_p95": 5.78, - "tokens_per_sec_max": 5.94 + "tokens_per_sec_max": 5.94, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92845.33, + "duration_ms_p50": 110870.0, + "tokens_per_sec_mean": 5.26 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81762.11, + "duration_ms_p50": 88383.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 17637.5, + "duration_ms_p50": 17538.5, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.55 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 99966.0, + "duration_ms_p50": 110822.0, + "tokens_per_sec_mean": 5.4, + "tokens_per_sec_p50": 5.38 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 135995.25, + "duration_ms_p50": 151381.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.23 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1265.92, + "answer_chars_mean": 766.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 16242, + "tokens_per_sec": 5.294914419406477, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18234, + "tokens_per_sec": 5.6487879785017, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 19231, + "tokens_per_sec": 5.45993448078623, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16843, + "tokens_per_sec": 5.937184587068812, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110870, + "tokens_per_sec": 5.276449896274916, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 110774, + "tokens_per_sec": 5.3712965136223305, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 112101, + "tokens_per_sec": 5.379077795916182, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66119, + "tokens_per_sec": 5.580846655273068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 151424, + "tokens_per_sec": 5.197326711749788, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 152836, + "tokens_per_sec": 5.227825904891517, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 88383, + "tokens_per_sec": 5.555366982338232, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 151338, + "tokens_per_sec": 5.226711070583726, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx131k", @@ -113,7 +463,182 @@ "tokens_per_sec_mean": 4.79, "tokens_per_sec_p50": 4.8, "tokens_per_sec_p95": 4.93, - "tokens_per_sec_max": 4.93 + "tokens_per_sec_max": 4.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 107527.33, + "duration_ms_p50": 120915.0, + "tokens_per_sec_mean": 4.69 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 86488.89, + "duration_ms_p50": 96577.0, + "tokens_per_sec_mean": 4.82 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34153.25, + "duration_ms_p50": 37396.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 4.79 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 98701.0, + "duration_ms_p50": 100326.5, + "tokens_per_sec_mean": 4.82, + "tokens_per_sec_p50": 4.8 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 142391.25, + "duration_ms_p50": 151248.0, + "tokens_per_sec_mean": 4.8, + "tokens_per_sec_p50": 4.79 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1250.42, + "answer_chars_mean": 714.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 34389, + "tokens_per_sec": 4.449097095001308, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18091, + "tokens_per_sec": 4.698468851915317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 40404, + "tokens_per_sec": 4.875754875754875, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43729, + "tokens_per_sec": 4.9166457042237415, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 120915, + "tokens_per_sec": 4.920812140760038, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 73236, + "tokens_per_sec": 4.765415915669889, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 104076, + "tokens_per_sec": 4.82339828586802, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 96577, + "tokens_per_sec": 4.783747683195792, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 167278, + "tokens_per_sec": 4.698764930235895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145866, + "tokens_per_sec": 4.744080183181824, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 99791, + "tokens_per_sec": 4.93030433606237, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 156630, + "tokens_per_sec": 4.839430505011811, + "error": null + } + ] } ], "n_calls_total": 45, @@ -140,7 +665,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -157,6 +683,181 @@ "tokens_per_sec_p50": 5.35, "tokens_per_sec_p95": 5.91, "tokens_per_sec_max": 5.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 88941.0, + "duration_ms_p50": 81140.0, + "tokens_per_sec_mean": 5.21 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 78787.89, + "duration_ms_p50": 70839.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22599.5, + "duration_ms_p50": 19202.0, + "tokens_per_sec_mean": 5.54, + "tokens_per_sec_p50": 5.54 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 71981.0, + "duration_ms_p50": 73215.0, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 149398.0, + "duration_ms_p50": 146891.5, + "tokens_per_sec_mean": 5.23, + "tokens_per_sec_p50": 5.26 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1030.83, + "answer_chars_mean": 891.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 38170, + "tokens_per_sec": 5.187319884726224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 15973, + "tokens_per_sec": 5.884930820760032, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 13824, + "tokens_per_sec": 5.931712962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 22431, + "tokens_per_sec": 5.171414560206856, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 81140, + "tokens_per_sec": 5.385753019472516, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 70839, + "tokens_per_sec": 5.533674953062579, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 60354, + "tokens_per_sec": 5.550584882526428, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 75591, + "tokens_per_sec": 5.5033006574856795, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 147513, + "tokens_per_sec": 5.063960464501434, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 157591, + "tokens_per_sec": 5.254107150789068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 146270, + "tokens_per_sec": 5.271074041156765, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 146218, + "tokens_per_sec": 5.313983230518815, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -411,6 +1112,181 @@ "tokens_per_sec_p50": 5.38, "tokens_per_sec_p95": 5.78, "tokens_per_sec_max": 5.94, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92845.33, + "duration_ms_p50": 110870.0, + "tokens_per_sec_mean": 5.26 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81762.11, + "duration_ms_p50": 88383.0, + "tokens_per_sec_mean": 5.49 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 17637.5, + "duration_ms_p50": 17538.5, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.55 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 99966.0, + "duration_ms_p50": 110822.0, + "tokens_per_sec_mean": 5.4, + "tokens_per_sec_p50": 5.38 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 135995.25, + "duration_ms_p50": 151381.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.23 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1265.92, + "answer_chars_mean": 766.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 16242, + "tokens_per_sec": 5.294914419406477, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18234, + "tokens_per_sec": 5.6487879785017, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 19231, + "tokens_per_sec": 5.45993448078623, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 16843, + "tokens_per_sec": 5.937184587068812, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110870, + "tokens_per_sec": 5.276449896274916, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 110774, + "tokens_per_sec": 5.3712965136223305, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 112101, + "tokens_per_sec": 5.379077795916182, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66119, + "tokens_per_sec": 5.580846655273068, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 151424, + "tokens_per_sec": 5.197326711749788, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 152836, + "tokens_per_sec": 5.227825904891517, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 88383, + "tokens_per_sec": 5.555366982338232, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 151338, + "tokens_per_sec": 5.226711070583726, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -665,6 +1541,181 @@ "tokens_per_sec_p50": 4.8, "tokens_per_sec_p95": 4.93, "tokens_per_sec_max": 4.93, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 107527.33, + "duration_ms_p50": 120915.0, + "tokens_per_sec_mean": 4.69 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 86488.89, + "duration_ms_p50": 96577.0, + "tokens_per_sec_mean": 4.82 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34153.25, + "duration_ms_p50": 37396.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 4.79 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 98701.0, + "duration_ms_p50": 100326.5, + "tokens_per_sec_mean": 4.82, + "tokens_per_sec_p50": 4.8 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 142391.25, + "duration_ms_p50": 151248.0, + "tokens_per_sec_mean": 4.8, + "tokens_per_sec_p50": 4.79 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1250.42, + "answer_chars_mean": 714.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 34389, + "tokens_per_sec": 4.449097095001308, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18091, + "tokens_per_sec": 4.698468851915317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 40404, + "tokens_per_sec": 4.875754875754875, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43729, + "tokens_per_sec": 4.9166457042237415, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 120915, + "tokens_per_sec": 4.920812140760038, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 73236, + "tokens_per_sec": 4.765415915669889, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 104076, + "tokens_per_sec": 4.82339828586802, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 96577, + "tokens_per_sec": 4.783747683195792, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 167278, + "tokens_per_sec": 4.698764930235895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145866, + "tokens_per_sec": 4.744080183181824, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 99791, + "tokens_per_sec": 4.93030433606237, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 156630, + "tokens_per_sec": 4.839430505011811, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -942,5 +1993,5 @@ "P-HARD": 2048 } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/86241185-24da-4f8e-b254-a60c9465e353/metadata.json b/runs/86241185-24da-4f8e-b254-a60c9465e353/metadata.json index cd56d97..2aa3c95 100644 --- a/runs/86241185-24da-4f8e-b254-a60c9465e353/metadata.json +++ b/runs/86241185-24da-4f8e-b254-a60c9465e353/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T03:07:15Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -30,7 +30,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-cpu-matrix-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -109,7 +109,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -121,7 +125,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -133,7 +141,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -145,7 +157,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -171,8 +187,10 @@ "metadata": "/data/86241185-24da-4f8e-b254-a60c9465e353/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/86241185-24da-4f8e-b254-a60c9465e353/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -188,6 +206,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -201,6 +223,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -214,6 +240,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -227,6 +257,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -253,5 +287,5 @@ "qwen2.5-72b" ] }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json b/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json index 317b2ff..989f52a 100644 --- a/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json +++ b/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T21:24:11Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -24,7 +24,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#predator-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -55,7 +55,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -78,8 +82,10 @@ "metadata": "/data/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -95,6 +101,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -117,5 +127,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/91751afd-068a-477b-8f40-6e1963f803f1/metadata.json b/runs/91751afd-068a-477b-8f40-6e1963f803f1/metadata.json index 9239f57..b623fd9 100644 --- a/runs/91751afd-068a-477b-8f40-6e1963f803f1/metadata.json +++ b/runs/91751afd-068a-477b-8f40-6e1963f803f1/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T10:23:13Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -126,7 +126,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -138,7 +142,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -150,7 +158,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -162,7 +174,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b-it-q4km-cpu-ctx32k", @@ -174,7 +190,37 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 2, @@ -203,7 +249,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -220,6 +267,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -233,6 +284,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -246,6 +301,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -259,6 +318,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -272,6 +335,36 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "ram_snapshot", @@ -311,5 +404,5 @@ "qwen2.5-72b" ] }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json b/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json index 50cf3de..45e8311 100644 --- a/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json +++ b/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json @@ -41,7 +41,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -312,7 +312,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", @@ -324,7 +399,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", @@ -336,7 +486,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", @@ -348,7 +573,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", @@ -360,7 +660,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", @@ -372,7 +747,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", @@ -384,7 +834,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", @@ -396,7 +921,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", @@ -408,7 +1008,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 6, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", @@ -420,7 +1095,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:14b", @@ -432,7 +1182,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:8b", @@ -444,7 +1269,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:4b", @@ -456,7 +1356,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", @@ -468,7 +1443,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", @@ -480,7 +1530,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", @@ -492,7 +1617,82 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 96, @@ -522,7 +1722,7 @@ "data_url": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/run.jsonl" }, "site_grade": "archive-only", - "site_grade_reason": "no documented method (no synthesis doc and no run.md)", + "site_grade_reason": "error rate 12/96", "raw_data_urls": { "jsonl": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/run.jsonl", "log": null, @@ -530,8 +1730,9 @@ "metadata": "/data/ad057f5b-ed3f-4a95-a38e-361be310ffd6/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6/" }, - "synthesis_doc": "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md", + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md" ], "related_ids": [], @@ -549,6 +1750,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -760,6 +2036,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -973,6 +2324,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1186,6 +2612,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1398,6 +2899,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1610,6 +3186,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1827,6 +3478,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -2044,6 +3770,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -2257,6 +4058,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 6, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "" + } + ], "calls": [ { "type": "call", @@ -2449,6 +4325,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -2667,6 +4618,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -2872,6 +4898,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -3084,6 +5185,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -3276,6 +5452,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -3493,6 +5744,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -3711,6 +6037,81 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 6, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 1, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 2, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 3, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 4, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -3964,5 +6365,5 @@ "env_inference_route": null, "env_llamacpp_url": null }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/ad28cb95-5134-40b5-8514-c8a381f83d87/metadata.json b/runs/ad28cb95-5134-40b5-8514-c8a381f83d87/metadata.json index c1974ae..55e9e1f 100644 --- a/runs/ad28cb95-5134-40b5-8514-c8a381f83d87/metadata.json +++ b/runs/ad28cb95-5134-40b5-8514-c8a381f83d87/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-ctx-sweep-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T23:48:01Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -89,7 +89,182 @@ "tokens_per_sec_mean": 4.69, "tokens_per_sec_p50": 4.74, "tokens_per_sec_p95": 4.92, - "tokens_per_sec_max": 4.95 + "tokens_per_sec_max": 4.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 90812.0, + "duration_ms_p50": 69978.0, + "tokens_per_sec_mean": 4.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 79562.67, + "duration_ms_p50": 68607.0, + "tokens_per_sec_mean": 4.79 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25406.0, + "duration_ms_p50": 24753.0, + "tokens_per_sec_mean": 4.61, + "tokens_per_sec_p50": 4.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 70149.25, + "duration_ms_p50": 69292.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 151569.75, + "duration_ms_p50": 167047.0, + "tokens_per_sec_mean": 4.7, + "tokens_per_sec_p50": 4.66 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1116.17, + "answer_chars_mean": 630.5 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 33949, + "tokens_per_sec": 3.7998173731185014, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 21340, + "tokens_per_sec": 4.779756326148079, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 28166, + "tokens_per_sec": 4.899524249094653, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18169, + "tokens_per_sec": 4.953492212009467, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 69978, + "tokens_per_sec": 4.730058018234303, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 75274, + "tokens_per_sec": 4.742673433057895, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 68607, + "tokens_per_sec": 4.8828836707624586, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66738, + "tokens_per_sec": 4.689981719560071, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 168509, + "tokens_per_sec": 4.664439288109241, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 173316, + "tokens_per_sec": 4.644695238754645, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98869, + "tokens_per_sec": 4.834680233440209, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 165585, + "tokens_per_sec": 4.662258054775493, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx64k", @@ -101,7 +276,182 @@ "tokens_per_sec_mean": 5.46, "tokens_per_sec_p50": 5.41, "tokens_per_sec_p95": 5.73, - "tokens_per_sec_max": 5.83 + "tokens_per_sec_max": 5.83, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 68214.0, + "duration_ms_p50": 89809.0, + "tokens_per_sec_mean": 5.3 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 82623.44, + "duration_ms_p50": 77959.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 26140.75, + "duration_ms_p50": 24427.0, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.61 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 80091.75, + "duration_ms_p50": 85924.0, + "tokens_per_sec_mean": 5.5, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 130830.75, + "duration_ms_p50": 142584.5, + "tokens_per_sec_mean": 5.28, + "tokens_per_sec_p50": 5.29 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1055.92, + "answer_chars_mean": 843.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19560, + "tokens_per_sec": 5.316973415132924, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 29294, + "tokens_per_sec": 5.564279374615962, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 37869, + "tokens_per_sec": 5.651060233964457, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17840, + "tokens_per_sec": 5.829596412556054, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 95273, + "tokens_per_sec": 5.374030417851857, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 93889, + "tokens_per_sec": 5.453247984321912, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 77959, + "tokens_per_sec": 5.579856078194949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 53246, + "tokens_per_sec": 5.577883784697442, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 89809, + "tokens_per_sec": 5.199924283757753, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145855, + "tokens_per_sec": 5.3340646532515175, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 148345, + "tokens_per_sec": 5.244531328996596, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 139314, + "tokens_per_sec": 5.347631968072125, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx128k", @@ -113,7 +463,182 @@ "tokens_per_sec_mean": 4.79, "tokens_per_sec_p50": 4.77, "tokens_per_sec_p95": 5.0, - "tokens_per_sec_max": 5.12 + "tokens_per_sec_max": 5.12, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 94769.67, + "duration_ms_p50": 97143.0, + "tokens_per_sec_mean": 4.64 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 87045.33, + "duration_ms_p50": 98125.0, + "tokens_per_sec_mean": 4.84 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25067.0, + "duration_ms_p50": 21998.5, + "tokens_per_sec_mean": 4.72, + "tokens_per_sec_p50": 4.76 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 101212.25, + "duration_ms_p50": 100751.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 140650.0, + "duration_ms_p50": 149948.0, + "tokens_per_sec_mean": 4.88, + "tokens_per_sec_p50": 4.83 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1302.08, + "answer_chars_mean": 611.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 22587, + "tokens_per_sec": 4.471598707220968, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 42874, + "tokens_per_sec": 4.898073424453048, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 21410, + "tokens_per_sec": 4.810836057916862, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13397, + "tokens_per_sec": 4.702545345972979, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97143, + "tokens_per_sec": 4.66322843642877, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 107841, + "tokens_per_sec": 4.747730455021745, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95505, + "tokens_per_sec": 4.732736505942097, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104360, + "tokens_per_sec": 4.906094288999617, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 164579, + "tokens_per_sec": 4.794050273728725, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 161591, + "tokens_per_sec": 4.746551478733345, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98125, + "tokens_per_sec": 5.115923566878981, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 138305, + "tokens_per_sec": 4.858826506633888, + "error": null + } + ] } ], "n_calls_total": 45, @@ -140,7 +665,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -157,6 +683,181 @@ "tokens_per_sec_p50": 4.74, "tokens_per_sec_p95": 4.92, "tokens_per_sec_max": 4.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 90812.0, + "duration_ms_p50": 69978.0, + "tokens_per_sec_mean": 4.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 79562.67, + "duration_ms_p50": 68607.0, + "tokens_per_sec_mean": 4.79 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25406.0, + "duration_ms_p50": 24753.0, + "tokens_per_sec_mean": 4.61, + "tokens_per_sec_p50": 4.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 70149.25, + "duration_ms_p50": 69292.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 151569.75, + "duration_ms_p50": 167047.0, + "tokens_per_sec_mean": 4.7, + "tokens_per_sec_p50": 4.66 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1116.17, + "answer_chars_mean": 630.5 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 33949, + "tokens_per_sec": 3.7998173731185014, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 21340, + "tokens_per_sec": 4.779756326148079, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 28166, + "tokens_per_sec": 4.899524249094653, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18169, + "tokens_per_sec": 4.953492212009467, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 69978, + "tokens_per_sec": 4.730058018234303, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 75274, + "tokens_per_sec": 4.742673433057895, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 68607, + "tokens_per_sec": 4.8828836707624586, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 66738, + "tokens_per_sec": 4.689981719560071, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 168509, + "tokens_per_sec": 4.664439288109241, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 173316, + "tokens_per_sec": 4.644695238754645, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98869, + "tokens_per_sec": 4.834680233440209, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 165585, + "tokens_per_sec": 4.662258054775493, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -411,6 +1112,181 @@ "tokens_per_sec_p50": 5.41, "tokens_per_sec_p95": 5.73, "tokens_per_sec_max": 5.83, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 68214.0, + "duration_ms_p50": 89809.0, + "tokens_per_sec_mean": 5.3 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 82623.44, + "duration_ms_p50": 77959.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 26140.75, + "duration_ms_p50": 24427.0, + "tokens_per_sec_mean": 5.59, + "tokens_per_sec_p50": 5.61 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 80091.75, + "duration_ms_p50": 85924.0, + "tokens_per_sec_mean": 5.5, + "tokens_per_sec_p50": 5.52 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 130830.75, + "duration_ms_p50": 142584.5, + "tokens_per_sec_mean": 5.28, + "tokens_per_sec_p50": 5.29 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1055.92, + "answer_chars_mean": 843.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19560, + "tokens_per_sec": 5.316973415132924, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 29294, + "tokens_per_sec": 5.564279374615962, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 37869, + "tokens_per_sec": 5.651060233964457, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 17840, + "tokens_per_sec": 5.829596412556054, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 95273, + "tokens_per_sec": 5.374030417851857, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 93889, + "tokens_per_sec": 5.453247984321912, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 77959, + "tokens_per_sec": 5.579856078194949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 53246, + "tokens_per_sec": 5.577883784697442, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 89809, + "tokens_per_sec": 5.199924283757753, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 145855, + "tokens_per_sec": 5.3340646532515175, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 148345, + "tokens_per_sec": 5.244531328996596, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 139314, + "tokens_per_sec": 5.347631968072125, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -665,6 +1541,181 @@ "tokens_per_sec_p50": 4.77, "tokens_per_sec_p95": 5.0, "tokens_per_sec_max": 5.12, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 94769.67, + "duration_ms_p50": 97143.0, + "tokens_per_sec_mean": 4.64 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 87045.33, + "duration_ms_p50": 98125.0, + "tokens_per_sec_mean": 4.84 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25067.0, + "duration_ms_p50": 21998.5, + "tokens_per_sec_mean": 4.72, + "tokens_per_sec_p50": 4.76 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 101212.25, + "duration_ms_p50": 100751.5, + "tokens_per_sec_mean": 4.76, + "tokens_per_sec_p50": 4.74 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 140650.0, + "duration_ms_p50": 149948.0, + "tokens_per_sec_mean": 4.88, + "tokens_per_sec_p50": 4.83 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1302.08, + "answer_chars_mean": 611.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 22587, + "tokens_per_sec": 4.471598707220968, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 42874, + "tokens_per_sec": 4.898073424453048, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 21410, + "tokens_per_sec": 4.810836057916862, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13397, + "tokens_per_sec": 4.702545345972979, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 97143, + "tokens_per_sec": 4.66322843642877, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 107841, + "tokens_per_sec": 4.747730455021745, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95505, + "tokens_per_sec": 4.732736505942097, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104360, + "tokens_per_sec": 4.906094288999617, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 164579, + "tokens_per_sec": 4.794050273728725, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 161591, + "tokens_per_sec": 4.746551478733345, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 98125, + "tokens_per_sec": 5.115923566878981, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 138305, + "tokens_per_sec": 4.858826506633888, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -937,5 +1988,5 @@ ], "gguf": "Qwen3-30B-A3B-UD-IQ2_M.gguf" }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07/metadata.json b/runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07/metadata.json index 540b184..32e9bee 100644 --- a/runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07/metadata.json +++ b/runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-ngl-matrix-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T22:50:47Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -89,7 +89,174 @@ "tokens_per_sec_mean": 4.95, "tokens_per_sec_p50": 4.98, "tokens_per_sec_p95": 5.15, - "tokens_per_sec_max": 5.15 + "tokens_per_sec_max": 5.15, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 95785.67, + "duration_ms_p50": 108039.0, + "tokens_per_sec_mean": 4.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81986.67, + "duration_ms_p50": 92416.0, + "tokens_per_sec_mean": 4.99 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 24673.5, + "duration_ms_p50": 23088.0, + "tokens_per_sec_mean": 4.84, + "tokens_per_sec_p50": 4.75 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 93075.75, + "duration_ms_p50": 96497.0, + "tokens_per_sec_mean": 4.9, + "tokens_per_sec_p50": 4.88 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 138560.0, + "duration_ms_p50": 153138.0, + "tokens_per_sec_mean": 5.12, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1263.58, + "answer_chars_mean": 656.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 23904, + "tokens_per_sec": 4.769076305220883, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 19868, + "tokens_per_sec": 4.731226092208577, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 22272, + "tokens_per_sec": 4.714439655172414, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 32650, + "tokens_per_sec": 5.145482388973966, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 108039, + "tokens_per_sec": 4.73902942455965, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 71270, + "tokens_per_sec": 4.910902202890417, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92416, + "tokens_per_sec": 4.858466066481995, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 100578, + "tokens_per_sec": 5.0905764680148735, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 155414, + "tokens_per_sec": 5.057459430939298, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92550, + "tokens_per_sec": 5.143165856293895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 153066, + "tokens_per_sec": 5.121973527759267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 153210, + "tokens_per_sec": 5.149794399843352, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl12", @@ -101,7 +268,174 @@ "tokens_per_sec_mean": 5.59, "tokens_per_sec_p50": 5.36, "tokens_per_sec_p95": 6.61, - "tokens_per_sec_max": 6.63 + "tokens_per_sec_max": 6.63, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 85208.33, + "duration_ms_p50": 64718.0, + "tokens_per_sec_mean": 5.44 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 71615.33, + "duration_ms_p50": 92482.0, + "tokens_per_sec_mean": 5.64 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20134.0, + "duration_ms_p50": 14613.0, + "tokens_per_sec_mean": 6.15, + "tokens_per_sec_p50": 6.26 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 81221.25, + "duration_ms_p50": 82069.0, + "tokens_per_sec_mean": 5.31, + "tokens_per_sec_p50": 5.31 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 123685.5, + "duration_ms_p50": 126628.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.3 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1129.25, + "answer_chars_mean": 657.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 41903, + "tokens_per_sec": 5.4411378660239125, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10852, + "tokens_per_sec": 6.634721710283818, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 9407, + "tokens_per_sec": 6.590836611034336, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18374, + "tokens_per_sec": 5.932295635136606, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 64718, + "tokens_per_sec": 5.67075620383819, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 96029, + "tokens_per_sec": 5.331722708765061, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 93926, + "tokens_per_sec": 4.971999233439091, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70212, + "tokens_per_sec": 5.28399703754344, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 149004, + "tokens_per_sec": 5.194491423049046, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92482, + "tokens_per_sec": 5.319954153240631, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 108851, + "tokens_per_sec": 5.282450321999798, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 144405, + "tokens_per_sec": 5.387625082233995, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl24", @@ -113,7 +447,174 @@ "tokens_per_sec_mean": 6.58, "tokens_per_sec_p50": 6.68, "tokens_per_sec_p95": 6.88, - "tokens_per_sec_max": 6.89 + "tokens_per_sec_max": 6.89, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66010.0, + "duration_ms_p50": 67078.0, + "tokens_per_sec_mean": 6.23 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 61473.11, + "duration_ms_p50": 67020.0, + "tokens_per_sec_mean": 6.7 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 12523.5, + "duration_ms_p50": 12403.5, + "tokens_per_sec_mean": 6.59, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66233.75, + "duration_ms_p50": 67049.0, + "tokens_per_sec_mean": 6.67, + "tokens_per_sec_p50": 6.69 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109064.75, + "duration_ms_p50": 118280.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.44 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1095.0, + "answer_chars_mean": 736.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 11689, + "tokens_per_sec": 5.731884677902301, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14951, + "tokens_per_sec": 6.889171292890108, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 10336, + "tokens_per_sec": 6.8691950464396285, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13118, + "tokens_per_sec": 6.860801951517, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 67078, + "tokens_per_sec": 6.57443573153642, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59196, + "tokens_per_sec": 6.689641191972431, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 71641, + "tokens_per_sec": 6.6861154925252295, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 67020, + "tokens_per_sec": 6.714413607878246, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 119263, + "tokens_per_sec": 6.397625416097196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 72039, + "tokens_per_sec": 6.676938880328711, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 117297, + "tokens_per_sec": 6.462228360486628, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127660, + "tokens_per_sec": 6.423311922293593, + "error": null + } + ] } ], "n_calls_total": 42, @@ -141,7 +642,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_NGL_RETUNE_2026-05-05.md" + "A3B_NGL_RETUNE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -158,6 +660,173 @@ "tokens_per_sec_p50": 4.98, "tokens_per_sec_p95": 5.15, "tokens_per_sec_max": 5.15, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 95785.67, + "duration_ms_p50": 108039.0, + "tokens_per_sec_mean": 4.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 81986.67, + "duration_ms_p50": 92416.0, + "tokens_per_sec_mean": 4.99 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 24673.5, + "duration_ms_p50": 23088.0, + "tokens_per_sec_mean": 4.84, + "tokens_per_sec_p50": 4.75 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 93075.75, + "duration_ms_p50": 96497.0, + "tokens_per_sec_mean": 4.9, + "tokens_per_sec_p50": 4.88 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 138560.0, + "duration_ms_p50": 153138.0, + "tokens_per_sec_mean": 5.12, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1263.58, + "answer_chars_mean": 656.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 23904, + "tokens_per_sec": 4.769076305220883, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 19868, + "tokens_per_sec": 4.731226092208577, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 22272, + "tokens_per_sec": 4.714439655172414, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 32650, + "tokens_per_sec": 5.145482388973966, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 108039, + "tokens_per_sec": 4.73902942455965, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 71270, + "tokens_per_sec": 4.910902202890417, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92416, + "tokens_per_sec": 4.858466066481995, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 100578, + "tokens_per_sec": 5.0905764680148735, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 155414, + "tokens_per_sec": 5.057459430939298, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92550, + "tokens_per_sec": 5.143165856293895, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 153066, + "tokens_per_sec": 5.121973527759267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 153210, + "tokens_per_sec": 5.149794399843352, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -390,6 +1059,173 @@ "tokens_per_sec_p50": 5.36, "tokens_per_sec_p95": 6.61, "tokens_per_sec_max": 6.63, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 85208.33, + "duration_ms_p50": 64718.0, + "tokens_per_sec_mean": 5.44 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 71615.33, + "duration_ms_p50": 92482.0, + "tokens_per_sec_mean": 5.64 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20134.0, + "duration_ms_p50": 14613.0, + "tokens_per_sec_mean": 6.15, + "tokens_per_sec_p50": 6.26 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 81221.25, + "duration_ms_p50": 82069.0, + "tokens_per_sec_mean": 5.31, + "tokens_per_sec_p50": 5.31 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 123685.5, + "duration_ms_p50": 126628.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.3 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1129.25, + "answer_chars_mean": 657.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 41903, + "tokens_per_sec": 5.4411378660239125, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10852, + "tokens_per_sec": 6.634721710283818, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 9407, + "tokens_per_sec": 6.590836611034336, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 18374, + "tokens_per_sec": 5.932295635136606, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 64718, + "tokens_per_sec": 5.67075620383819, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 96029, + "tokens_per_sec": 5.331722708765061, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 93926, + "tokens_per_sec": 4.971999233439091, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 70212, + "tokens_per_sec": 5.28399703754344, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 149004, + "tokens_per_sec": 5.194491423049046, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92482, + "tokens_per_sec": 5.319954153240631, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 108851, + "tokens_per_sec": 5.282450321999798, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 144405, + "tokens_per_sec": 5.387625082233995, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -622,6 +1458,173 @@ "tokens_per_sec_p50": 6.68, "tokens_per_sec_p95": 6.88, "tokens_per_sec_max": 6.89, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66010.0, + "duration_ms_p50": 67078.0, + "tokens_per_sec_mean": 6.23 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 61473.11, + "duration_ms_p50": 67020.0, + "tokens_per_sec_mean": 6.7 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 12523.5, + "duration_ms_p50": 12403.5, + "tokens_per_sec_mean": 6.59, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66233.75, + "duration_ms_p50": 67049.0, + "tokens_per_sec_mean": 6.67, + "tokens_per_sec_p50": 6.69 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109064.75, + "duration_ms_p50": 118280.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.44 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1095.0, + "answer_chars_mean": 736.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 11689, + "tokens_per_sec": 5.731884677902301, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14951, + "tokens_per_sec": 6.889171292890108, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 10336, + "tokens_per_sec": 6.8691950464396285, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13118, + "tokens_per_sec": 6.860801951517, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 67078, + "tokens_per_sec": 6.57443573153642, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59196, + "tokens_per_sec": 6.689641191972431, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 71641, + "tokens_per_sec": 6.6861154925252295, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 67020, + "tokens_per_sec": 6.714413607878246, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 119263, + "tokens_per_sec": 6.397625416097196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 72039, + "tokens_per_sec": 6.676938880328711, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 117297, + "tokens_per_sec": 6.462228360486628, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127660, + "tokens_per_sec": 6.423311922293593, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -864,5 +1867,5 @@ "sampling": "T=0.6 top_p=0.95 top_k=20 (Qwen3-A3B README defaults)", "common_args": "--ctx-size 32768 --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.0 --reasoning-budget 500 --reasoning-format deepseek --jinja --no-warmup --host 0.0.0.0 --port 11436" }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json b/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json index d1fa250..bef18ad 100644 --- a/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json +++ b/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-04", "started_at": "2026-05-04T23:53:09Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -22,7 +22,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#vps50-gemma-e4b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -53,7 +53,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -76,8 +80,10 @@ "metadata": "/data/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -93,6 +99,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -112,5 +122,5 @@ "ngl": 0, "ctx": 32768 }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f/metadata.json b/runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f/metadata.json index 985c62a..e191b85 100644 --- a/runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f/metadata.json +++ b/runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f/metadata.json @@ -3,7 +3,7 @@ "title": "vps50-cpu-matrix-1 \u2014 gemma/phi/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "started_at": "2026-05-05T07:00:10Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "vps50", "hardware_label": "VPS50 \u00b7 cloud \u00b7 16 vCPU AMD EPYC \u00b7 62 GB RAM \u00b7 no GPU", "engine": "llamacpp", @@ -125,7 +125,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", @@ -137,7 +141,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", @@ -149,7 +157,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", @@ -161,7 +173,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] }, { "cell_id": "vps50:llamacpp:phi-4-q4km-cpu-ctx32k", @@ -173,7 +189,94 @@ "tokens_per_sec_mean": 0.02, "tokens_per_sec_p50": 0.02, "tokens_per_sec_p95": 0.02, - "tokens_per_sec_max": 0.02 + "tokens_per_sec_max": 0.02, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": 540002.0, + "duration_ms_p50": 540002.0, + "tokens_per_sec_mean": 0.02 + }, + "warm": { + "n_calls": 3, + "duration_ms_mean": 551918.67, + "duration_ms_p50": 515603.0, + "tokens_per_sec_mean": 0.02 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 548939.5, + "duration_ms_p50": 527802.5, + "tokens_per_sec_mean": 0.02, + "tokens_per_sec_p50": 0.02 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 34.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 540002, + "tokens_per_sec": 0.01851844993166692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 628077, + "tokens_per_sec": 0.015921614706477072, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 512076, + "tokens_per_sec": 0.01952835126035979, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 515603, + "tokens_per_sec": 0.019394766903993967, + "error": null + } + ] } ], "n_calls_total": 6, @@ -202,7 +305,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -219,6 +323,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -232,6 +340,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -245,6 +357,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -258,6 +374,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] }, { @@ -271,6 +391,93 @@ "tokens_per_sec_p50": 0.02, "tokens_per_sec_p95": 0.02, "tokens_per_sec_max": 0.02, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": 540002.0, + "duration_ms_p50": 540002.0, + "tokens_per_sec_mean": 0.02 + }, + "warm": { + "n_calls": 3, + "duration_ms_mean": 551918.67, + "duration_ms_p50": 515603.0, + "tokens_per_sec_mean": 0.02 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 548939.5, + "duration_ms_p50": 527802.5, + "tokens_per_sec_mean": 0.02, + "tokens_per_sec_p50": 0.02 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 34.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 540002, + "tokens_per_sec": 0.01851844993166692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 628077, + "tokens_per_sec": 0.015921614706477072, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 512076, + "tokens_per_sec": 0.01952835126035979, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 515603, + "tokens_per_sec": 0.019394766903993967, + "error": null + } + ], "calls": [ { "type": "ram_snapshot", @@ -374,5 +581,5 @@ "qwen2.5-72b" ] }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json b/runs/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json index 131fc2e..4eab2ef 100644 --- a/runs/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json +++ b/runs/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json @@ -3,7 +3,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T22:02:28Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "pavilion", "hardware_label": "Pavilion \u00b7 HP laptop \u00b7 GTX 1050 4 GB \u00b7 16 GB RAM \u00b7 i7-9750H", "engine": "llamacpp", @@ -24,7 +24,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#pavilion-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -55,7 +55,11 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [] } ], "n_calls_total": 0, @@ -78,8 +82,10 @@ "metadata": "/data/d1cff064-9141-48e0-b1d2-729b134e0543/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/d1cff064-9141-48e0-b1d2-729b134e0543/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "meta-only", "visibility": "draft", @@ -95,6 +101,10 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": null, + "chars_split": null, + "timeline": [], "calls": [] } ], @@ -119,5 +129,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/e08a7051-a856-4053-b34b-561d7ce5e8cf/metadata.json b/runs/e08a7051-a856-4053-b34b-561d7ce5e8cf/metadata.json index 9348080..7d36f98 100644 --- a/runs/e08a7051-a856-4053-b34b-561d7ce5e8cf/metadata.json +++ b/runs/e08a7051-a856-4053-b34b-561d7ce5e8cf/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-ngl-ctx-2d-1 \u2014 qwen3 on predator", "date": "2026-05-05", "started_at": "2026-05-05T07:43:00Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -137,7 +137,182 @@ "tokens_per_sec_mean": 5.48, "tokens_per_sec_p50": 5.31, "tokens_per_sec_p95": 6.79, - "tokens_per_sec_max": 6.8 + "tokens_per_sec_max": 6.8, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92828.67, + "duration_ms_p50": 110762.0, + "tokens_per_sec_mean": 5.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 83155.0, + "duration_ms_p50": 76258.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18907.5, + "duration_ms_p50": 19953.0, + "tokens_per_sec_mean": 6.04, + "tokens_per_sec_p50": 6.21 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 84750.25, + "duration_ms_p50": 78940.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.32 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 153062.5, + "duration_ms_p50": 153912.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.14 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1193.67, + "answer_chars_mean": 809.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 25689, + "tokens_per_sec": 5.644439254155475, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10035, + "tokens_per_sec": 6.776283009466866, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15585, + "tokens_per_sec": 6.801411613731151, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 24321, + "tokens_per_sec": 4.934007647711853, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110762, + "tokens_per_sec": 5.29062313789928, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 76258, + "tokens_per_sec": 5.101104146450209, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70359, + "tokens_per_sec": 5.344021376085505, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 81622, + "tokens_per_sec": 5.451961480973267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 142035, + "tokens_per_sec": 5.252226563875102, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 150027, + "tokens_per_sec": 5.339038972984863, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 162391, + "tokens_per_sec": 5.024909015893738, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 157797, + "tokens_per_sec": 4.835326400375165, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx96k", @@ -149,7 +324,182 @@ "tokens_per_sec_mean": 5.79, "tokens_per_sec_p50": 5.75, "tokens_per_sec_p95": 6.47, - "tokens_per_sec_max": 6.7 + "tokens_per_sec_max": 6.7, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 65655.0, + "duration_ms_p50": 45721.0, + "tokens_per_sec_mean": 5.92 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 73856.11, + "duration_ms_p50": 59728.0, + "tokens_per_sec_mean": 5.75 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20742.5, + "duration_ms_p50": 18111.0, + "tokens_per_sec_mean": 5.82, + "tokens_per_sec_p50": 5.83 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66764.5, + "duration_ms_p50": 58541.0, + "tokens_per_sec_mean": 5.84, + "tokens_per_sec_p50": 5.76 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 127910.5, + "duration_ms_p50": 132742.5, + "tokens_per_sec_mean": 5.71, + "tokens_per_sec_p50": 5.72 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1209.5, + "answer_chars_mean": 618.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 13701, + "tokens_per_sec": 5.766002481570688, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33047, + "tokens_per_sec": 4.932369050140709, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15830, + "tokens_per_sec": 6.69614655716993, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20392, + "tokens_per_sec": 5.884660651235779, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 45721, + "tokens_per_sec": 6.277203035804117, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59728, + "tokens_per_sec": 5.575274578087329, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 57354, + "tokens_per_sec": 5.666562053213377, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104255, + "tokens_per_sec": 5.85103831950506, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 137543, + "tokens_per_sec": 5.729117439637059, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 144293, + "tokens_per_sec": 5.516553124545196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 101864, + "tokens_per_sec": 5.703683342495877, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127942, + "tokens_per_sec": 5.893295399477889, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx131k", @@ -161,7 +511,182 @@ "tokens_per_sec_mean": 5.01, "tokens_per_sec_p50": 5.13, "tokens_per_sec_p95": 5.97, - "tokens_per_sec_max": 6.29 + "tokens_per_sec_max": 6.29, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109523.67, + "duration_ms_p50": 118125.0, + "tokens_per_sec_mean": 4.11 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 92033.67, + "duration_ms_p50": 95486.0, + "tokens_per_sec_mean": 5.31 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 33574.75, + "duration_ms_p50": 31872.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 5.19 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 103544.75, + "duration_ms_p50": 105842.0, + "tokens_per_sec_mean": 5.19, + "tokens_per_sec_p50": 5.05 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 152099.0, + "duration_ms_p50": 150187.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1296.25, + "answer_chars_mean": 873.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 59744, + "tokens_per_sec": 2.259641135511516, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10810, + "tokens_per_sec": 6.290471785383904, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 20002, + "tokens_per_sec": 5.049495050494951, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43743, + "tokens_per_sec": 5.326566536360103, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 118125, + "tokens_per_sec": 4.960846560846561, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 116198, + "tokens_per_sec": 5.146388061756657, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95486, + "tokens_per_sec": 4.932660285277422, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 84370, + "tokens_per_sec": 5.71293113665995, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 150702, + "tokens_per_sec": 5.122692465926132, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 148897, + "tokens_per_sec": 5.131063755481978, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 159125, + "tokens_per_sec": 4.788688138256088, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 149672, + "tokens_per_sec": 5.378427494788604, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx48k", @@ -173,7 +698,182 @@ "tokens_per_sec_mean": 6.6, "tokens_per_sec_p50": 6.59, "tokens_per_sec_p95": 6.89, - "tokens_per_sec_max": 6.91 + "tokens_per_sec_max": 6.91, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66671.33, + "duration_ms_p50": 58574.0, + "tokens_per_sec_mean": 6.36 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 65162.22, + "duration_ms_p50": 69317.0, + "tokens_per_sec_mean": 6.68 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16306.0, + "duration_ms_p50": 15313.0, + "tokens_per_sec_mean": 6.71, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 75223.75, + "duration_ms_p50": 74910.0, + "tokens_per_sec_mean": 6.61, + "tokens_per_sec_p50": 6.59 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 105088.75, + "duration_ms_p50": 114731.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.48 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1152.67, + "answer_chars_mean": 769.33 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19864, + "tokens_per_sec": 6.192106322996375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14734, + "tokens_per_sec": 6.854893443735578, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15636, + "tokens_per_sec": 6.907137375287798, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14990, + "tokens_per_sec": 6.871247498332221, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 58574, + "tokens_per_sec": 6.572882166148803, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 89924, + "tokens_per_sec": 6.60557804368133, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92501, + "tokens_per_sec": 6.551280526697008, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 59896, + "tokens_per_sec": 6.728329103779885, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 121576, + "tokens_per_sec": 6.317036257156018, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 69317, + "tokens_per_sec": 6.6794581415814305, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 114261, + "tokens_per_sec": 6.485152414209573, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 115201, + "tokens_per_sec": 6.475638232307012, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx64k", @@ -185,7 +885,182 @@ "tokens_per_sec_mean": 6.41, "tokens_per_sec_p50": 6.39, "tokens_per_sec_p95": 6.88, - "tokens_per_sec_max": 6.9 + "tokens_per_sec_max": 6.9, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 80477.67, + "duration_ms_p50": 99471.0, + "tokens_per_sec_mean": 5.99 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 68197.56, + "duration_ms_p50": 69604.0, + "tokens_per_sec_mean": 6.55 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16514.0, + "duration_ms_p50": 16585.0, + "tokens_per_sec_mean": 6.53, + "tokens_per_sec_p50": 6.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 79436.25, + "duration_ms_p50": 81054.0, + "tokens_per_sec_mean": 6.43, + "tokens_per_sec_p50": 6.44 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 117852.5, + "duration_ms_p50": 124494.0, + "tokens_per_sec_mean": 6.25, + "tokens_per_sec_p50": 6.25 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1063.5, + "answer_chars_mean": 944.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 18535, + "tokens_per_sec": 5.557054221742649, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18478, + "tokens_per_sec": 6.873038207598224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 14692, + "tokens_per_sec": 6.806425265450585, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14351, + "tokens_per_sec": 6.898473973939098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 99471, + "tokens_per_sec": 6.2430256054528455, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 56166, + "tokens_per_sec": 6.445180358223837, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92504, + "tokens_per_sec": 6.442964628556602, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 69604, + "tokens_per_sec": 6.5944485949083385, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 123427, + "tokens_per_sec": 6.165587756325601, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 90927, + "tokens_per_sec": 6.345749887272207, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 131495, + "tokens_per_sec": 6.319631925168257, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 125561, + "tokens_per_sec": 6.188227236164095, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx96k", @@ -197,7 +1072,182 @@ "tokens_per_sec_mean": 6.08, "tokens_per_sec_p50": 6.44, "tokens_per_sec_p95": 6.68, - "tokens_per_sec_max": 6.69 + "tokens_per_sec_max": 6.69, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 78350.33, + "duration_ms_p50": 72290.0, + "tokens_per_sec_mean": 4.93 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 66282.56, + "duration_ms_p50": 64420.0, + "tokens_per_sec_mean": 6.47 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25171.0, + "duration_ms_p50": 25926.5, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 6.3 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 73270.5, + "duration_ms_p50": 68355.0, + "tokens_per_sec_mean": 6.39, + "tokens_per_sec_p50": 6.46 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109457.0, + "duration_ms_p50": 117967.0, + "tokens_per_sec_mean": 6.37, + "tokens_per_sec_p50": 6.4 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1185.42, + "answer_chars_mean": 741.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35465, + "tokens_per_sec": 2.7068941209643307, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 27111, + "tokens_per_sec": 6.1598613109070115, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 24742, + "tokens_per_sec": 6.668822245574327, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13366, + "tokens_per_sec": 6.43423612150232, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 72290, + "tokens_per_sec": 5.97593028081339, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92055, + "tokens_per_sec": 6.474390310140676, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 64317, + "tokens_per_sec": 6.436867391202949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 64420, + "tokens_per_sec": 6.690468798509779, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 127296, + "tokens_per_sec": 6.1038838612368025, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 123105, + "tokens_per_sec": 6.344177734454328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 74598, + "tokens_per_sec": 6.568540711547227, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 112829, + "tokens_per_sec": 6.45224188816705, + "error": null + } + ] } ], "n_calls_total": 90, @@ -227,7 +1277,8 @@ }, "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ - "A3B_AND_CPU_OVERNIGHT_2026-05-05.md" + "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -244,6 +1295,181 @@ "tokens_per_sec_p50": 5.31, "tokens_per_sec_p95": 6.79, "tokens_per_sec_max": 6.8, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 92828.67, + "duration_ms_p50": 110762.0, + "tokens_per_sec_mean": 5.4 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 83155.0, + "duration_ms_p50": 76258.0, + "tokens_per_sec_mean": 5.51 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 18907.5, + "duration_ms_p50": 19953.0, + "tokens_per_sec_mean": 6.04, + "tokens_per_sec_p50": 6.21 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 84750.25, + "duration_ms_p50": 78940.0, + "tokens_per_sec_mean": 5.3, + "tokens_per_sec_p50": 5.32 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 153062.5, + "duration_ms_p50": 153912.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.14 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1193.67, + "answer_chars_mean": 809.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 25689, + "tokens_per_sec": 5.644439254155475, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10035, + "tokens_per_sec": 6.776283009466866, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15585, + "tokens_per_sec": 6.801411613731151, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 24321, + "tokens_per_sec": 4.934007647711853, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 110762, + "tokens_per_sec": 5.29062313789928, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 76258, + "tokens_per_sec": 5.101104146450209, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 70359, + "tokens_per_sec": 5.344021376085505, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 81622, + "tokens_per_sec": 5.451961480973267, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 142035, + "tokens_per_sec": 5.252226563875102, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 150027, + "tokens_per_sec": 5.339038972984863, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 162391, + "tokens_per_sec": 5.024909015893738, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 157797, + "tokens_per_sec": 4.835326400375165, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -498,6 +1724,181 @@ "tokens_per_sec_p50": 5.75, "tokens_per_sec_p95": 6.47, "tokens_per_sec_max": 6.7, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 65655.0, + "duration_ms_p50": 45721.0, + "tokens_per_sec_mean": 5.92 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 73856.11, + "duration_ms_p50": 59728.0, + "tokens_per_sec_mean": 5.75 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 20742.5, + "duration_ms_p50": 18111.0, + "tokens_per_sec_mean": 5.82, + "tokens_per_sec_p50": 5.83 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 66764.5, + "duration_ms_p50": 58541.0, + "tokens_per_sec_mean": 5.84, + "tokens_per_sec_p50": 5.76 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 127910.5, + "duration_ms_p50": 132742.5, + "tokens_per_sec_mean": 5.71, + "tokens_per_sec_p50": 5.72 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1209.5, + "answer_chars_mean": 618.75 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 13701, + "tokens_per_sec": 5.766002481570688, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33047, + "tokens_per_sec": 4.932369050140709, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15830, + "tokens_per_sec": 6.69614655716993, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20392, + "tokens_per_sec": 5.884660651235779, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 45721, + "tokens_per_sec": 6.277203035804117, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59728, + "tokens_per_sec": 5.575274578087329, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 57354, + "tokens_per_sec": 5.666562053213377, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 104255, + "tokens_per_sec": 5.85103831950506, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 137543, + "tokens_per_sec": 5.729117439637059, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 144293, + "tokens_per_sec": 5.516553124545196, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 101864, + "tokens_per_sec": 5.703683342495877, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 127942, + "tokens_per_sec": 5.893295399477889, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -752,6 +2153,181 @@ "tokens_per_sec_p50": 5.13, "tokens_per_sec_p95": 5.97, "tokens_per_sec_max": 6.29, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 109523.67, + "duration_ms_p50": 118125.0, + "tokens_per_sec_mean": 4.11 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 92033.67, + "duration_ms_p50": 95486.0, + "tokens_per_sec_mean": 5.31 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 33574.75, + "duration_ms_p50": 31872.5, + "tokens_per_sec_mean": 4.73, + "tokens_per_sec_p50": 5.19 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 103544.75, + "duration_ms_p50": 105842.0, + "tokens_per_sec_mean": 5.19, + "tokens_per_sec_p50": 5.05 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 152099.0, + "duration_ms_p50": 150187.0, + "tokens_per_sec_mean": 5.11, + "tokens_per_sec_p50": 5.13 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1296.25, + "answer_chars_mean": 873.42 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 59744, + "tokens_per_sec": 2.259641135511516, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 10810, + "tokens_per_sec": 6.290471785383904, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 20002, + "tokens_per_sec": 5.049495050494951, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 43743, + "tokens_per_sec": 5.326566536360103, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 118125, + "tokens_per_sec": 4.960846560846561, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 116198, + "tokens_per_sec": 5.146388061756657, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 95486, + "tokens_per_sec": 4.932660285277422, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 84370, + "tokens_per_sec": 5.71293113665995, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 150702, + "tokens_per_sec": 5.122692465926132, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 148897, + "tokens_per_sec": 5.131063755481978, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 159125, + "tokens_per_sec": 4.788688138256088, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 149672, + "tokens_per_sec": 5.378427494788604, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -1006,6 +2582,181 @@ "tokens_per_sec_p50": 6.59, "tokens_per_sec_p95": 6.89, "tokens_per_sec_max": 6.91, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 66671.33, + "duration_ms_p50": 58574.0, + "tokens_per_sec_mean": 6.36 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 65162.22, + "duration_ms_p50": 69317.0, + "tokens_per_sec_mean": 6.68 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16306.0, + "duration_ms_p50": 15313.0, + "tokens_per_sec_mean": 6.71, + "tokens_per_sec_p50": 6.86 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 75223.75, + "duration_ms_p50": 74910.0, + "tokens_per_sec_mean": 6.61, + "tokens_per_sec_p50": 6.59 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 105088.75, + "duration_ms_p50": 114731.0, + "tokens_per_sec_mean": 6.49, + "tokens_per_sec_p50": 6.48 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1152.67, + "answer_chars_mean": 769.33 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 19864, + "tokens_per_sec": 6.192106322996375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 14734, + "tokens_per_sec": 6.854893443735578, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 15636, + "tokens_per_sec": 6.907137375287798, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14990, + "tokens_per_sec": 6.871247498332221, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 58574, + "tokens_per_sec": 6.572882166148803, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 89924, + "tokens_per_sec": 6.60557804368133, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92501, + "tokens_per_sec": 6.551280526697008, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 59896, + "tokens_per_sec": 6.728329103779885, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 121576, + "tokens_per_sec": 6.317036257156018, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 69317, + "tokens_per_sec": 6.6794581415814305, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 114261, + "tokens_per_sec": 6.485152414209573, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 115201, + "tokens_per_sec": 6.475638232307012, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -1260,6 +3011,181 @@ "tokens_per_sec_p50": 6.39, "tokens_per_sec_p95": 6.88, "tokens_per_sec_max": 6.9, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 80477.67, + "duration_ms_p50": 99471.0, + "tokens_per_sec_mean": 5.99 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 68197.56, + "duration_ms_p50": 69604.0, + "tokens_per_sec_mean": 6.55 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 16514.0, + "duration_ms_p50": 16585.0, + "tokens_per_sec_mean": 6.53, + "tokens_per_sec_p50": 6.84 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 79436.25, + "duration_ms_p50": 81054.0, + "tokens_per_sec_mean": 6.43, + "tokens_per_sec_p50": 6.44 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 117852.5, + "duration_ms_p50": 124494.0, + "tokens_per_sec_mean": 6.25, + "tokens_per_sec_p50": 6.25 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1063.5, + "answer_chars_mean": 944.25 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 18535, + "tokens_per_sec": 5.557054221742649, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 18478, + "tokens_per_sec": 6.873038207598224, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 14692, + "tokens_per_sec": 6.806425265450585, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 14351, + "tokens_per_sec": 6.898473973939098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 99471, + "tokens_per_sec": 6.2430256054528455, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 56166, + "tokens_per_sec": 6.445180358223837, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 92504, + "tokens_per_sec": 6.442964628556602, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 69604, + "tokens_per_sec": 6.5944485949083385, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 123427, + "tokens_per_sec": 6.165587756325601, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 90927, + "tokens_per_sec": 6.345749887272207, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 131495, + "tokens_per_sec": 6.319631925168257, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 125561, + "tokens_per_sec": 6.188227236164095, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -1514,6 +3440,181 @@ "tokens_per_sec_p50": 6.44, "tokens_per_sec_p95": 6.68, "tokens_per_sec_max": 6.69, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 78350.33, + "duration_ms_p50": 72290.0, + "tokens_per_sec_mean": 4.93 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 66282.56, + "duration_ms_p50": 64420.0, + "tokens_per_sec_mean": 6.47 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 3, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 25171.0, + "duration_ms_p50": 25926.5, + "tokens_per_sec_mean": 5.49, + "tokens_per_sec_p50": 6.3 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 73270.5, + "duration_ms_p50": 68355.0, + "tokens_per_sec_mean": 6.39, + "tokens_per_sec_p50": 6.46 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 109457.0, + "duration_ms_p50": 117967.0, + "tokens_per_sec_mean": 6.37, + "tokens_per_sec_p50": 6.4 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1185.42, + "answer_chars_mean": 741.92 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35465, + "tokens_per_sec": 2.7068941209643307, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 27111, + "tokens_per_sec": 6.1598613109070115, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 24742, + "tokens_per_sec": 6.668822245574327, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 13366, + "tokens_per_sec": 6.43423612150232, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 72290, + "tokens_per_sec": 5.97593028081339, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 92055, + "tokens_per_sec": 6.474390310140676, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 64317, + "tokens_per_sec": 6.436867391202949, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 64420, + "tokens_per_sec": 6.690468798509779, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 127296, + "tokens_per_sec": 6.1038838612368025, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 123105, + "tokens_per_sec": 6.344177734454328, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 74598, + "tokens_per_sec": 6.568540711547227, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 112829, + "tokens_per_sec": 6.45224188816705, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -1807,5 +3908,5 @@ }, "methodology_notes": "Sloba 2026-05-05: 'max tokens should be bigger than reasoning budget'. Prior bench had hello max_tokens=256 < reasoning_budget=500 \u2192 content_chars=0. Bumped to 512/1024/2048 so reasoning + answer both fit." }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d/metadata.json b/runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d/metadata.json index a39bf2c..f17579d 100644 --- a/runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d/metadata.json +++ b/runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d/metadata.json @@ -3,7 +3,7 @@ "title": "Predator Qwen rerun", "date": "2026-05-04", "started_at": "2026-05-04T17:49:07Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -88,7 +88,174 @@ "tokens_per_sec_mean": 14.25, "tokens_per_sec_p50": 14.91, "tokens_per_sec_p95": 15.3, - "tokens_per_sec_max": 15.32 + "tokens_per_sec_max": 15.32, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 32096.67, + "duration_ms_p50": 35949.0, + "tokens_per_sec_mean": 12.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 29933.56, + "duration_ms_p50": 33586.0, + "tokens_per_sec_mean": 14.72 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 5178.25, + "duration_ms_p50": 4671.5, + "tokens_per_sec_mean": 12.69, + "tokens_per_sec_p50": 13.7 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34217.0, + "duration_ms_p50": 33717.5, + "tokens_per_sec_mean": 14.98, + "tokens_per_sec_p50": 15.19 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 52027.75, + "duration_ms_p50": 51815.5, + "tokens_per_sec_mean": 15.09, + "tokens_per_sec_p50": 15.07 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1456.67, + "answer_chars_mean": 434.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6773, + "tokens_per_sec": 9.449283921452828, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4597, + "tokens_per_sec": 13.922123123776375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4632, + "tokens_per_sec": 13.816925734024181, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4711, + "tokens_per_sec": 13.585226066652515, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35949, + "tokens_per_sec": 14.242398954073828, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33849, + "tokens_per_sec": 15.126000768117228, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 33586, + "tokens_per_sec": 15.244447091049842, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 33484, + "tokens_per_sec": 15.290885198900966, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 53568, + "tokens_per_sec": 14.878285543608126, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 51802, + "tokens_per_sec": 14.941508049882245, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 51829, + "tokens_per_sec": 15.203843408130583, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 50912, + "tokens_per_sec": 15.320553111250787, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km-nothink", @@ -100,7 +267,174 @@ "tokens_per_sec_mean": 12.55, "tokens_per_sec_p50": 14.37, "tokens_per_sec_p95": 14.93, - "tokens_per_sec_max": 14.95 + "tokens_per_sec_max": 14.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 10193.33, + "duration_ms_p50": 7338.0, + "tokens_per_sec_mean": 11.56 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 10503.22, + "duration_ms_p50": 7062.0, + "tokens_per_sec_mean": 12.89 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1597.75, + "duration_ms_p50": 1393.0, + "tokens_per_sec_mean": 8.47, + "tokens_per_sec_p50": 9.34 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 7203.75, + "duration_ms_p50": 7200.0, + "tokens_per_sec_mean": 14.32, + "tokens_per_sec_p50": 14.37 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22475.75, + "duration_ms_p50": 22106.5, + "tokens_per_sec_mean": 14.87, + "tokens_per_sec_p50": 14.88 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 678.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 2372, + "tokens_per_sec": 5.480607082630692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 1350, + "tokens_per_sec": 9.62962962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 1436, + "tokens_per_sec": 9.052924791086351, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 1233, + "tokens_per_sec": 9.7323600973236, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 7338, + "tokens_per_sec": 14.445352957209048, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 8034, + "tokens_per_sec": 14.438635797859098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 7062, + "tokens_per_sec": 14.301897479467572, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6381, + "tokens_per_sec": 14.104372355430183, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20870, + "tokens_per_sec": 14.758025874460948, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 24820, + "tokens_per_sec": 14.947622884770347, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 23327, + "tokens_per_sec": 14.918334976636514, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20886, + "tokens_per_sec": 14.842478215072298, + "error": null + } + ] }, { "cell_id": "predator:llamacpp:qwen3:14b-q4km", @@ -112,7 +446,118 @@ "tokens_per_sec_mean": 1.06, "tokens_per_sec_p50": 1.09, "tokens_per_sec_p95": 1.28, - "tokens_per_sec_max": 1.33 + "tokens_per_sec_max": 1.33, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": 198723.5, + "duration_ms_p50": 198723.5, + "tokens_per_sec_mean": 0.9 + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": 96952.5, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 1.14 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 68859.25, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 0.97, + "tokens_per_sec_p50": 1.08 + }, + "P-MEDIUM": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": 254910.0, + "duration_ms_p50": 254910.0, + "tokens_per_sec_mean": 1.24, + "tokens_per_sec_p50": 1.24 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 504.5, + "answer_chars_mean": 201.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 98037, + "tokens_per_sec": 0.6528147536134317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59855, + "tokens_per_sec": 1.0692506891654834, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 58654, + "tokens_per_sec": 1.0911446789647765, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58891, + "tokens_per_sec": 1.0867534937426773, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 299410, + "tokens_per_sec": 1.142246417955312, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 210410, + "tokens_per_sec": 1.3307352312152465, + "error": null + } + ] } ], "n_calls_total": 36, @@ -140,7 +585,8 @@ "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "synthesis_docs_all": [ "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", - "A3B_CROSS_MACHINE_2026-05-05.md" + "A3B_CROSS_MACHINE_2026-05-05.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" ], "related_ids": [], "status": "complete", @@ -157,6 +603,173 @@ "tokens_per_sec_p50": 14.91, "tokens_per_sec_p95": 15.3, "tokens_per_sec_max": 15.32, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 32096.67, + "duration_ms_p50": 35949.0, + "tokens_per_sec_mean": 12.86 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 29933.56, + "duration_ms_p50": 33586.0, + "tokens_per_sec_mean": 14.72 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 5178.25, + "duration_ms_p50": 4671.5, + "tokens_per_sec_mean": 12.69, + "tokens_per_sec_p50": 13.7 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 34217.0, + "duration_ms_p50": 33717.5, + "tokens_per_sec_mean": 14.98, + "tokens_per_sec_p50": 15.19 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 52027.75, + "duration_ms_p50": 51815.5, + "tokens_per_sec_mean": 15.09, + "tokens_per_sec_p50": 15.07 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 1456.67, + "answer_chars_mean": 434.17 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 6773, + "tokens_per_sec": 9.449283921452828, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 4597, + "tokens_per_sec": 13.922123123776375, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 4632, + "tokens_per_sec": 13.816925734024181, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 4711, + "tokens_per_sec": 13.585226066652515, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 35949, + "tokens_per_sec": 14.242398954073828, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 33849, + "tokens_per_sec": 15.126000768117228, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 33586, + "tokens_per_sec": 15.244447091049842, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 33484, + "tokens_per_sec": 15.290885198900966, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 53568, + "tokens_per_sec": 14.878285543608126, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 51802, + "tokens_per_sec": 14.941508049882245, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 51829, + "tokens_per_sec": 15.203843408130583, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 50912, + "tokens_per_sec": 15.320553111250787, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -375,6 +988,173 @@ "tokens_per_sec_p50": 14.37, "tokens_per_sec_p95": 14.93, "tokens_per_sec_max": 14.95, + "cold_warm_split": { + "cold": { + "n_calls": 3, + "duration_ms_mean": 10193.33, + "duration_ms_p50": 7338.0, + "tokens_per_sec_mean": 11.56 + }, + "warm": { + "n_calls": 9, + "duration_ms_mean": 10503.22, + "duration_ms_p50": 7062.0, + "tokens_per_sec_mean": 12.89 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 1597.75, + "duration_ms_p50": 1393.0, + "tokens_per_sec_mean": 8.47, + "tokens_per_sec_p50": 9.34 + }, + "P-MEDIUM": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 7203.75, + "duration_ms_p50": 7200.0, + "tokens_per_sec_mean": 14.32, + "tokens_per_sec_p50": 14.37 + }, + "P-HARD": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 22475.75, + "duration_ms_p50": 22106.5, + "tokens_per_sec_mean": 14.87, + "tokens_per_sec_p50": 14.88 + } + }, + "chars_split": { + "has_thinking": false, + "reasoning_chars_mean": 0.0, + "answer_chars_mean": 678.58 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 2372, + "tokens_per_sec": 5.480607082630692, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 1350, + "tokens_per_sec": 9.62962962962963, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 1436, + "tokens_per_sec": 9.052924791086351, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 1233, + "tokens_per_sec": 9.7323600973236, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 7338, + "tokens_per_sec": 14.445352957209048, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 8034, + "tokens_per_sec": 14.438635797859098, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 2, + "phase": "warm", + "duration_ms": 7062, + "tokens_per_sec": 14.301897479467572, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 3, + "phase": "warm", + "duration_ms": 6381, + "tokens_per_sec": 14.104372355430183, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 0, + "phase": "cold", + "duration_ms": 20870, + "tokens_per_sec": 14.758025874460948, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 1, + "phase": "warm", + "duration_ms": 24820, + "tokens_per_sec": 14.947622884770347, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 2, + "phase": "warm", + "duration_ms": 23327, + "tokens_per_sec": 14.918334976636514, + "error": null + }, + { + "prompt_id": "P-HARD", + "run_idx": 3, + "phase": "warm", + "duration_ms": 20886, + "tokens_per_sec": 14.842478215072298, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -593,6 +1373,117 @@ "tokens_per_sec_p50": 1.09, "tokens_per_sec_p95": 1.28, "tokens_per_sec_max": 1.33, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": 198723.5, + "duration_ms_p50": 198723.5, + "tokens_per_sec_mean": 0.9 + }, + "warm": { + "n_calls": 4, + "duration_ms_mean": 96952.5, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 1.14 + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + }, + "hello": { + "n_calls": 4, + "n_errors": 0, + "duration_ms_mean": 68859.25, + "duration_ms_p50": 59373.0, + "tokens_per_sec_mean": 0.97, + "tokens_per_sec_p50": 1.08 + }, + "P-MEDIUM": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": 254910.0, + "duration_ms_p50": 254910.0, + "tokens_per_sec_mean": 1.24, + "tokens_per_sec_p50": 1.24 + } + }, + "chars_split": { + "has_thinking": true, + "reasoning_chars_mean": 504.5, + "answer_chars_mean": 201.0 + }, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 0, + "phase": "cold", + "duration_ms": 98037, + "tokens_per_sec": 0.6528147536134317, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 1, + "phase": "warm", + "duration_ms": 59855, + "tokens_per_sec": 1.0692506891654834, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 2, + "phase": "warm", + "duration_ms": 58654, + "tokens_per_sec": 1.0911446789647765, + "error": null + }, + { + "prompt_id": "hello", + "run_idx": 3, + "phase": "warm", + "duration_ms": 58891, + "tokens_per_sec": 1.0867534937426773, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 0, + "phase": "cold", + "duration_ms": 299410, + "tokens_per_sec": 1.142246417955312, + "error": null + }, + { + "prompt_id": "P-MEDIUM", + "run_idx": 1, + "phase": "warm", + "duration_ms": 210410, + "tokens_per_sec": 1.3307352312152465, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -735,5 +1626,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json b/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json index aaf0871..6666208 100644 --- a/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json +++ b/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json @@ -3,7 +3,7 @@ "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "started_at": "2026-05-04T18:50:19Z", - "git_sha": "f06800254dfd046e154c94f663fb7f9f66c79ae3", + "git_sha": "3ff8bd1808a1df00426d9b447559c774e4704c39", "hardware": "predator", "hardware_label": "Predator \u00b7 gaming laptop \u00b7 GTX 1060 6 GB \u00b7 28 GB RAM", "engine": "llamacpp", @@ -23,7 +23,7 @@ "janie_blurb_status": "pending", "caveat": null, "caveat_severity": null, - "methodology_ref": "HARNESS#predator-a3b-1", + "methodology_ref": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06", "methodology_deviations_md": null, "results_table": [ { @@ -54,7 +54,37 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 2, @@ -68,8 +98,8 @@ ], "data_url": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.jsonl" }, - "site_grade": "archive-only", - "site_grade_reason": "no documented method (no synthesis doc and no run.md)", + "site_grade": "standard", + "site_grade_reason": null, "raw_data_urls": { "jsonl": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.jsonl", "log": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/run.log", @@ -77,8 +107,10 @@ "metadata": "/data/fe4fa289-67a9-48e8-8a80-d5d88c875208/metadata.json", "gitea_dir": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public/src/branch/main/runs/fe4fa289-67a9-48e8-8a80-d5d88c875208/" }, - "synthesis_doc": null, - "synthesis_docs_all": [], + "synthesis_doc": "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", + "synthesis_docs_all": [ + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md" + ], "related_ids": [], "status": "complete", "visibility": "draft", @@ -94,6 +126,36 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": null, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": null, + "phase": null, + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "vram_snapshot", @@ -129,5 +191,5 @@ } } }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" } diff --git a/runs/ff1131ca-d021-4e06-8616-4b4cdb54e97e/metadata.json b/runs/ff1131ca-d021-4e06-8616-4b4cdb54e97e/metadata.json index be9ffb1..f522a70 100644 --- a/runs/ff1131ca-d021-4e06-8616-4b4cdb54e97e/metadata.json +++ b/runs/ff1131ca-d021-4e06-8616-4b4cdb54e97e/metadata.json @@ -312,7 +312,50 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", @@ -324,7 +367,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", @@ -336,7 +414,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", @@ -348,7 +461,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", @@ -360,7 +508,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", @@ -372,7 +555,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", @@ -384,7 +602,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", @@ -396,7 +649,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", @@ -408,7 +696,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", @@ -420,7 +743,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:14b", @@ -432,7 +790,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:8b", @@ -444,7 +837,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ] }, { "cell_id": "pavilion:weeyuga:qwen3:4b", @@ -456,7 +884,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", @@ -468,7 +931,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", @@ -480,7 +978,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", @@ -492,7 +1025,42 @@ "tokens_per_sec_mean": null, "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, - "tokens_per_sec_max": null + "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ] } ], "n_calls_total": 17, @@ -533,6 +1101,7 @@ "synthesis_doc": "PAVILION_WEEYUGA_v1.md", "synthesis_docs_all": [ "PAVILION_WEEYUGA_v1.md", + "SITE_DATA_AUDIT_AND_MIGRATION_PLAN_2026-05-06.md", "PAVILION_LLAMACPP_VS_OLLAMA_v0_INCOMPLETE.md" ], "related_ids": [], @@ -550,6 +1119,49 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 2, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 2, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + }, + { + "prompt_id": null, + "run_idx": 0, + "phase": "5q", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -623,6 +1235,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -662,6 +1309,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -701,6 +1383,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -740,6 +1457,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -779,6 +1531,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -818,6 +1605,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -857,6 +1679,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -896,6 +1753,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ], "calls": [ { "type": "call", @@ -935,6 +1827,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -974,6 +1901,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ], "calls": [ { "type": "call", @@ -1013,6 +1975,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": "TimeoutError('timed out')" + } + ], "calls": [ { "type": "call", @@ -1052,6 +2049,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1091,6 +2123,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1130,6 +2197,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1169,6 +2271,41 @@ "tokens_per_sec_p50": null, "tokens_per_sec_p95": null, "tokens_per_sec_max": null, + "cold_warm_split": { + "cold": { + "n_calls": 1, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + }, + "warm": { + "n_calls": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null + } + }, + "per_prompt": { + "unknown": { + "n_calls": 1, + "n_errors": 0, + "duration_ms_mean": null, + "duration_ms_p50": null, + "tokens_per_sec_mean": null, + "tokens_per_sec_p50": null + } + }, + "chars_split": null, + "timeline": [ + { + "prompt_id": null, + "run_idx": 0, + "phase": "hello", + "duration_ms": null, + "tokens_per_sec": null, + "error": null + } + ], "calls": [ { "type": "call", @@ -1243,5 +2380,5 @@ "env_inference_route": null, "env_llamacpp_url": null }, - "packaged_at": "2026-05-06T08:04:27Z" + "packaged_at": "2026-05-06T12:28:24Z" }