{ "schema_version": "1.0-draft", "generated_at": "2026-05-05T17:43:34Z", "git_sha": "d15ba45042cd26173e1696b3e47fd9e4fbf2613d", "site_base_url": "https://benchmarks.weeyuga.com", "data_base_url_lean": "https://benchmarks.weeyuga.com/data", "gitea_archive_url_lean": "https://git.weeyuga.com/slobodanmargetic988/weeyuga-benchmarks-public", "benchmarks": [ { "id": "91751afd-068a-477b-8f40-6e1963f803f1", "title": "vps50-cpu-matrix-1 \u2014 gemma/phi-4/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-cpu-matrix-1", "model_family": "gemma/phi-4/qwen2.5/qwen3", "model_sizes": [ "phi-4", "gemma-4-26b-a4b", "qwen3-30b-a3b", "qwen2.5-72b", "gemma-4-26b-a4b-it" ], "task_kind": null, "headline": "2 calls across 5 cell(s)", "summary_md_path": null, "raw_jsonl_path": "runs/91751afd-068a-477b-8f40-6e1963f803f1.jsonl", "log_path": "runs/91751afd-068a-477b-8f40-6e1963f803f1.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "vps50:llamacpp:phi-4", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b-it-q4km-cpu-ctx32k", "n_calls": 2, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 2, "n_errors_total": 0, "started_at": "2026-05-05T10:23:13Z", "tags": [ "a3b", "cpu", "gemma", "matrix", "qwen2.5", "qwen3", "vps50" ], "status": "complete", "visibility": "draft" }, { "id": "79afe366-1055-4e45-adf6-593864a530e8", "title": "predator-a3b-ngl6-ctx-1 \u2014 qwen3 on predator", "date": "2026-05-05", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-ngl6-ctx-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "45 calls across 3 cell(s); ~5.2 tok/s mean; p50 84.8s", "summary_md_path": null, "raw_jsonl_path": "runs/79afe366-1055-4e45-adf6-593864a530e8.jsonl", "log_path": "runs/79afe366-1055-4e45-adf6-593864a530e8.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx64k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 81326.17, "duration_ms_p50": 73215.0, "tokens_per_sec_mean": 5.42, "tokens_per_sec_max": 5.93 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx96k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 84532.92, "duration_ms_p50": 99578.5, "tokens_per_sec_mean": 5.43, "tokens_per_sec_max": 5.94 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx131k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 91748.5, "duration_ms_p50": 98184.0, "tokens_per_sec_mean": 4.79, "tokens_per_sec_max": 4.93 } ], "n_calls_total": 45, "n_errors_total": 0, "started_at": "2026-05-05T09:44:34Z", "tags": [ "a3b", "ctx", "iq2", "ngl", "predator", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "e08a7051-a856-4053-b34b-561d7ce5e8cf", "title": "predator-a3b-ngl-ctx-2d-1 \u2014 qwen3 on predator", "date": "2026-05-05", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-ngl-ctx-2d-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "90 calls across 6 cell(s); ~5.9 tok/s mean; p50 73.4s", "summary_md_path": null, "raw_jsonl_path": "runs/e08a7051-a856-4053-b34b-561d7ce5e8cf.jsonl", "log_path": "runs/e08a7051-a856-4053-b34b-561d7ce5e8cf.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx64k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 85573.42, "duration_ms_p50": 78940.0, "tokens_per_sec_mean": 5.48, "tokens_per_sec_max": 6.8 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx96k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 71805.83, "duration_ms_p50": 58541.0, "tokens_per_sec_mean": 5.79, "tokens_per_sec_max": 6.7 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl12-ctx131k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 96406.17, "duration_ms_p50": 105842.0, "tokens_per_sec_mean": 5.01, "tokens_per_sec_max": 6.29 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx48k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 65539.5, "duration_ms_p50": 64606.5, "tokens_per_sec_mean": 6.6, "tokens_per_sec_max": 6.91 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx64k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 71267.58, "duration_ms_p50": 80265.5, "tokens_per_sec_mean": 6.41, "tokens_per_sec_max": 6.9 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl24-ctx96k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 69299.5, "duration_ms_p50": 68355.0, "tokens_per_sec_mean": 6.08, "tokens_per_sec_max": 6.69 } ], "n_calls_total": 90, "n_errors_total": 0, "started_at": "2026-05-05T07:43:00Z", "tags": [ "a3b", "ctx", "iq2", "ngl", "predator", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "cef4b219-6b65-4740-8c3b-5b68aa10cf3f", "title": "vps50-cpu-matrix-1 \u2014 gemma/phi-4/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-cpu-matrix-1", "model_family": "gemma/phi-4/qwen2.5/qwen3", "model_sizes": [ "phi-4", "gemma-4-26b-a4b", "qwen3-30b-a3b", "qwen2.5-72b" ], "task_kind": null, "headline": "6 calls across 5 cell(s); ~0.0 tok/s mean; p50 527.8s", "summary_md_path": null, "raw_jsonl_path": "runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f.jsonl", "log_path": "runs/cef4b219-6b65-4740-8c3b-5b68aa10cf3f.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "vps50:llamacpp:phi-4", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:phi-4-q4km-cpu-ctx32k", "n_calls": 6, "n_errors": 0, "duration_ms_mean": 548939.5, "duration_ms_p50": 527802.5, "tokens_per_sec_mean": 0.02, "tokens_per_sec_max": 0.02 } ], "n_calls_total": 6, "n_errors_total": 0, "started_at": "2026-05-05T07:00:10Z", "tags": [ "a3b", "cpu", "gemma", "matrix", "qwen2.5", "qwen3", "vps50" ], "status": "complete", "visibility": "draft" }, { "id": "86241185-24da-4f8e-b254-a60c9465e353", "title": "vps50-cpu-matrix-1 \u2014 gemma/phi-4/qwen2.5/qwen3 on vps50", "date": "2026-05-05", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-cpu-matrix-1", "model_family": "gemma/phi-4/qwen2.5/qwen3", "model_sizes": [ "phi-4", "gemma-4-26b-a4b", "qwen3-30b-a3b", "qwen2.5-72b" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/86241185-24da-4f8e-b254-a60c9465e353.jsonl", "log_path": "runs/86241185-24da-4f8e-b254-a60c9465e353.log", "synthesis_doc": null, "cells": [ { "cell_id": "vps50:llamacpp:phi-4", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-05T03:07:15Z", "tags": [ "a3b", "cpu", "gemma", "matrix", "qwen2.5", "qwen3", "vps50" ], "status": "meta-only", "visibility": "draft" }, { "id": "1bf57c9a-fd7a-49aa-90de-cd1907b15ddd", "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-05", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-gemma-e4b-1", "model_family": "gemma", "model_sizes": [ "gemma-4-e4b" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd.jsonl", "log_path": "runs/1bf57c9a-fd7a-49aa-90de-cd1907b15ddd.log", "synthesis_doc": null, "cells": [ { "cell_id": "vps50:llamacpp:gemma-4-e4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-05T03:07:13Z", "tags": [ "gemma", "vps50" ], "status": "meta-only", "visibility": "draft" }, { "id": "b54c61c0-b6b8-44ac-bc3e-a515df0f0499", "title": "vps50-gemma-e4b-1 \u2014 gemma on vps50", "date": "2026-05-04", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-gemma-e4b-1", "model_family": "gemma", "model_sizes": [ "gemma-4-e4b" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499.jsonl", "log_path": "runs/b54c61c0-b6b8-44ac-bc3e-a515df0f0499.log", "synthesis_doc": null, "cells": [ { "cell_id": "vps50:llamacpp:gemma-4-e4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T23:53:09Z", "tags": [ "gemma", "vps50" ], "status": "meta-only", "visibility": "draft" }, { "id": "ad28cb95-5134-40b5-8514-c8a381f83d87", "title": "predator-a3b-ctx-sweep-1 \u2014 qwen3 on predator", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-ctx-sweep-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "45 calls across 3 cell(s); ~5.0 tok/s mean; p50 83.9s", "summary_md_path": null, "raw_jsonl_path": "runs/ad28cb95-5134-40b5-8514-c8a381f83d87.jsonl", "log_path": "runs/ad28cb95-5134-40b5-8514-c8a381f83d87.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl36-ctx32k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 82375.0, "duration_ms_p50": 69292.5, "tokens_per_sec_mean": 4.69, "tokens_per_sec_max": 4.95 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx64k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 79021.08, "duration_ms_p50": 83884.0, "tokens_per_sec_mean": 5.46, "tokens_per_sec_max": 5.83 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ngl6-ctx128k", "n_calls": 15, "n_errors": 0, "duration_ms_mean": 88976.42, "duration_ms_p50": 97634.0, "tokens_per_sec_mean": 4.79, "tokens_per_sec_max": 5.12 } ], "n_calls_total": 45, "n_errors_total": 0, "started_at": "2026-05-04T23:48:01Z", "tags": [ "a3b", "ctx", "iq2", "predator", "qwen3", "sweep" ], "status": "complete", "visibility": "draft" }, { "id": "212d6278-1b9b-45e9-8aae-7eed4d4ec822", "title": "vps50-cpu-matrix-1 \u2014 gemma/phi-4/qwen2.5/qwen3 on vps50", "date": "2026-05-04", "hardware": "vps50", "engine": "llamacpp", "harness": "vps50-cpu-matrix-1", "model_family": "gemma/phi-4/qwen2.5/qwen3", "model_sizes": [ "phi-4", "gemma-4-26b-a4b", "qwen3-30b-a3b", "qwen2.5-72b" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822.jsonl", "log_path": "runs/212d6278-1b9b-45e9-8aae-7eed4d4ec822.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "vps50:llamacpp:phi-4", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:gemma-4-26b-a4b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen3-30b-a3b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "vps50:llamacpp:qwen2.5-72b", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T23:39:53Z", "tags": [ "a3b", "cpu", "gemma", "matrix", "qwen2.5", "qwen3", "vps50" ], "status": "meta-only", "visibility": "draft" }, { "id": "b37836bf-d1a2-4d6b-a732-aff89da1fa07", "title": "predator-a3b-ngl-matrix-1 \u2014 qwen3 on predator", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-ngl-matrix-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "42 calls across 3 cell(s); ~5.7 tok/s mean; p50 71.5s", "summary_md_path": null, "raw_jsonl_path": "runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07.jsonl", "log_path": "runs/b37836bf-d1a2-4d6b-a732-aff89da1fa07.log", "synthesis_doc": "A3B_NGL_RETUNE_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl6", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 85436.42, "duration_ms_p50": 92483.0, "tokens_per_sec_mean": 4.95, "tokens_per_sec_max": 5.15 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl12", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 75013.58, "duration_ms_p50": 81347.0, "tokens_per_sec_mean": 5.59, "tokens_per_sec_max": 6.63 }, { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500-ctx32k-ngl24", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 62607.33, "duration_ms_p50": 67049.0, "tokens_per_sec_mean": 6.58, "tokens_per_sec_max": 6.89 } ], "n_calls_total": 42, "n_errors_total": 0, "started_at": "2026-05-04T22:50:47Z", "tags": [ "a3b", "iq2", "matrix", "ngl", "predator", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "23066b38-ea9c-4dd3-b2f5-32912a67fce4", "title": "Predator Qwen rerun \u2014 23066b38-ea9c-4dd3-b2f5-32912a67fce4", "date": "2026-05-04", "hardware": "pavilion", "engine": "llamacpp", "harness": "pavilion-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2xxs" ], "task_kind": null, "headline": "`Answer chars` is `len(message.content)`. Both averaged over the 3 warm runs.", "summary_md_path": "runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4.md", "raw_jsonl_path": "runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4.jsonl", "log_path": "runs/23066b38-ea9c-4dd3-b2f5-32912a67fce4.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "pavilion:llamacpp:qwen3:30b-a3b-iq2xxs-think500", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 72439.92, "duration_ms_p50": 57784.0, "tokens_per_sec_mean": 5.88, "tokens_per_sec_max": 8.19 } ], "n_calls_total": 14, "n_errors_total": 0, "started_at": "2026-05-04T22:11:43Z", "tags": [ "a3b", "iq2", "pavilion", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "d1cff064-9141-48e0-b1d2-729b134e0543", "title": "Predator Qwen rerun \u2014 d1cff064-9141-48e0-b1d2-729b134e0543", "date": "2026-05-04", "hardware": "pavilion", "engine": "llamacpp", "harness": "pavilion-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2xxs" ], "task_kind": null, "headline": "`Answer chars` is `len(message.content)`. Both averaged over the 3 warm runs.", "summary_md_path": "runs/d1cff064-9141-48e0-b1d2-729b134e0543.md", "raw_jsonl_path": "runs/d1cff064-9141-48e0-b1d2-729b134e0543.jsonl", "log_path": "runs/d1cff064-9141-48e0-b1d2-729b134e0543.log", "synthesis_doc": null, "cells": [ { "cell_id": "pavilion:llamacpp:qwen3:30b-a3b-iq2xxs-think500", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T22:02:28Z", "tags": [ "a3b", "iq2", "pavilion", "qwen3" ], "status": "meta-only", "visibility": "draft" }, { "id": "5fb2913d-6500-4ecf-9e97-d43f7dd61145", "title": "Predator Qwen rerun \u2014 5fb2913d-6500-4ecf-9e97-d43f7dd61145", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "`Answer chars` is `len(message.content)`. Both averaged over the 3 warm runs.", "summary_md_path": "runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145.md", "raw_jsonl_path": "runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145.jsonl", "log_path": "runs/5fb2913d-6500-4ecf-9e97-d43f7dd61145.log", "synthesis_doc": "A3B_NGL_RETUNE_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 107312.08, "duration_ms_p50": 101920.0, "tokens_per_sec_mean": 3.87, "tokens_per_sec_max": 4.06 } ], "n_calls_total": 14, "n_errors_total": 0, "started_at": "2026-05-04T22:02:17Z", "tags": [ "a3b", "iq2", "predator", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef", "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2m" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef.jsonl", "log_path": "runs/8ec3c8cb-58a7-4f33-974c-9c1bfb7824ef.log", "synthesis_doc": null, "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-iq2m-think500", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T21:24:11Z", "tags": [ "a3b", "iq2", "predator", "qwen3" ], "status": "meta-only", "visibility": "draft" }, { "id": "487c8a51-7c5c-417d-a0c5-4b6e8491e11f", "title": "Predator Qwen rerun \u2014 487c8a51-7c5c-417d-a0c5-4b6e8491e11f", "date": "2026-05-04", "hardware": "pavilion", "engine": "llamacpp", "harness": "pavilion-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2xxs" ], "task_kind": null, "headline": "`Answer chars` is `len(message.content)`. Both averaged over the 3 warm runs.", "summary_md_path": "runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f.md", "raw_jsonl_path": "runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f.jsonl", "log_path": "runs/487c8a51-7c5c-417d-a0c5-4b6e8491e11f.log", "synthesis_doc": null, "cells": [ { "cell_id": "pavilion:llamacpp:qwen3:30b-a3b-iq2xxs-think500", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T21:24:10Z", "tags": [ "a3b", "iq2", "pavilion", "qwen3" ], "status": "meta-only", "visibility": "draft" }, { "id": "5f4d9e97-4294-4e69-9e6c-f318bca1ce44", "title": "pavilion-a3b-1 \u2014 qwen3 on pavilion", "date": "2026-05-04", "hardware": "pavilion", "engine": "llamacpp", "harness": "pavilion-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b-iq2xxs" ], "task_kind": null, "headline": "no calls landed", "summary_md_path": null, "raw_jsonl_path": "runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44.jsonl", "log_path": "runs/5f4d9e97-4294-4e69-9e6c-f318bca1ce44.log", "synthesis_doc": null, "cells": [ { "cell_id": "pavilion:llamacpp:qwen3:30b-a3b-iq2xxs-think500", "n_calls": 0, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 0, "n_errors_total": 0, "started_at": "2026-05-04T20:38:52Z", "tags": [ "a3b", "iq2", "pavilion", "qwen3" ], "status": "meta-only", "visibility": "draft" }, { "id": "fe4fa289-67a9-48e8-8a80-d5d88c875208", "title": "predator-a3b-1 \u2014 qwen3 on predator", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-a3b-1", "model_family": "qwen3", "model_sizes": [ "qwen3:30b-a3b" ], "task_kind": null, "headline": "2 calls across 1 cell(s)", "summary_md_path": null, "raw_jsonl_path": "runs/fe4fa289-67a9-48e8-8a80-d5d88c875208.jsonl", "log_path": "runs/fe4fa289-67a9-48e8-8a80-d5d88c875208.log", "synthesis_doc": null, "cells": [ { "cell_id": "predator:llamacpp:qwen3:30b-a3b-q4km-think500", "n_calls": 2, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 2, "n_errors_total": 0, "started_at": "2026-05-04T18:50:19Z", "tags": [ "a3b", "predator", "qwen3" ], "status": "complete", "visibility": "draft" }, { "id": "fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", "title": "Predator Qwen rerun \u2014 fba9d9b1-cc5d-40bc-9e21-beafbb72c65d", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-qwen-rerun-1", "model_family": "qwen3/qwen3.5", "model_sizes": [ "qwen3.5:9b", "qwen3:14b" ], "task_kind": null, "headline": "`Answer chars` is `len(message.content)`. Both averaged over the 3 warm runs.", "summary_md_path": "runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d.md", "raw_jsonl_path": "runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d.jsonl", "log_path": "runs/fba9d9b1-cc5d-40bc-9e21-beafbb72c65d.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km-think500", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 30474.33, "duration_ms_p50": 33717.5, "tokens_per_sec_mean": 14.25, "tokens_per_sec_max": 15.32 }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km-nothink", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 10425.75, "duration_ms_p50": 7200.0, "tokens_per_sec_mean": 12.55, "tokens_per_sec_max": 14.95 }, { "cell_id": "predator:llamacpp:qwen3:14b-q4km", "n_calls": 8, "n_errors": 0, "duration_ms_mean": 130876.17, "duration_ms_p50": 78946.0, "tokens_per_sec_mean": 1.06, "tokens_per_sec_max": 1.33 } ], "n_calls_total": 36, "n_errors_total": 0, "started_at": "2026-05-04T17:49:07Z", "tags": [ "predator", "qwen3", "qwen3.5", "rerun" ], "status": "complete", "visibility": "draft" }, { "id": "09d8fbde-0008-49bb-99da-03eeaca72be1", "title": "Predator trio bench \u2014 09d8fbde-0008-49bb-99da-03eeaca72be1", "date": "2026-05-04", "hardware": "predator", "engine": "llamacpp", "harness": "predator-trio-1", "model_family": "gemma/granite/qwen3.5", "model_sizes": [ "granite-4.1:8b", "gemma-4:e4b-it", "qwen3.5:9b" ], "task_kind": null, "headline": "includes prefill time, so true generation rate is slightly higher. Use llama-bench's", "summary_md_path": "runs/09d8fbde-0008-49bb-99da-03eeaca72be1.md", "raw_jsonl_path": "runs/09d8fbde-0008-49bb-99da-03eeaca72be1.jsonl", "log_path": "runs/09d8fbde-0008-49bb-99da-03eeaca72be1.log", "synthesis_doc": "A3B_AND_CPU_OVERNIGHT_2026-05-05.md", "cells": [ { "cell_id": "predator:llamacpp:granite-4.1:8b-q4km", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 8831.33, "duration_ms_p50": 6403.0, "tokens_per_sec_mean": 13.49, "tokens_per_sec_max": 15.75 }, { "cell_id": "predator:llamacpp:gemma-4:e4b-it-q4km", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 9888.42, "duration_ms_p50": 8199.0, "tokens_per_sec_mean": 21.82, "tokens_per_sec_max": 23.66 }, { "cell_id": "predator:llamacpp:qwen3.5:9b-q4km", "n_calls": 14, "n_errors": 0, "duration_ms_mean": 37136.33, "duration_ms_p50": 35324.5, "tokens_per_sec_mean": 14.04, "tokens_per_sec_max": 14.57 } ], "n_calls_total": 42, "n_errors_total": 0, "started_at": "2026-05-04T16:01:52Z", "tags": [ "gemma", "granite", "predator", "qwen3", "qwen3.5", "trio" ], "status": "complete", "visibility": "draft" }, { "id": "ad057f5b-ed3f-4a95-a38e-361be310ffd6", "title": "pavilion-weeyuga-v3 \u2014 qwen2.5/qwen2.5-coder/qwen3/qwen3.5 on pavilion", "date": "2026-04-29", "hardware": "pavilion", "engine": "weeyuga", "harness": "pavilion-weeyuga-v3", "model_family": "qwen2.5/qwen2.5-coder/qwen3/qwen3.5", "model_sizes": [ "qwen3.5:4b", "qwen3.5:35b-a3b-uncensored-iq1m", "qwen3.5:35b-a3b-iq2s", "qwen3.5:9b-q6k", "qwen3.5:9b", "qwen3.5:2b", "qwen3.5:0.8b", "qwen2.5-coder:14b", "qwen2.5-coder:3b", "qwen3:14b", "qwen3:8b", "qwen3:4b", "qwen2.5:3b", "qwen2.5-coder:1.5b", "qwen2.5-coder:0.5b" ], "task_kind": null, "headline": "96 calls across 16 cell(s); 12 errors", "summary_md_path": null, "raw_jsonl_path": "runs/ad057f5b-ed3f-4a95-a38e-361be310ffd6.jsonl", "log_path": null, "synthesis_doc": null, "cells": [ { "cell_id": "pavilion:weeyuga:qwen3.5:4b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", "n_calls": 6, "n_errors": 6, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:14b", "n_calls": 6, "n_errors": 4, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:8b", "n_calls": 6, "n_errors": 2, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:4b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", "n_calls": 6, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 96, "n_errors_total": 12, "started_at": "2026-04-29T09:51:46Z", "tags": [ "a3b", "iq2", "pavilion", "qwen2.5", "qwen3", "qwen3.5", "weeyuga" ], "status": "complete", "visibility": "draft" }, { "id": "ff1131ca-d021-4e06-8616-4b4cdb54e97e", "title": "pavilion-weeyuga-v1 \u2014 qwen2.5/qwen2.5-coder/qwen3/qwen3.5 on pavilion", "date": "2026-04-28", "hardware": "pavilion", "engine": "weeyuga", "harness": "pavilion-weeyuga-v1", "model_family": "qwen2.5/qwen2.5-coder/qwen3/qwen3.5", "model_sizes": [ "qwen3.5:4b", "qwen3.5:35b-a3b-uncensored-iq1m", "qwen3.5:35b-a3b-iq2s", "qwen3.5:9b-q6k", "qwen3.5:9b", "qwen3.5:2b", "qwen3.5:0.8b", "qwen2.5-coder:14b", "qwen2.5-coder:3b", "qwen3:14b", "qwen3:8b", "qwen3:4b", "qwen2.5:3b", "qwen2.5-coder:1.5b", "qwen2.5-coder:0.5b" ], "task_kind": null, "headline": "17 calls across 16 cell(s); 3 errors", "summary_md_path": null, "raw_jsonl_path": "runs/ff1131ca-d021-4e06-8616-4b4cdb54e97e.jsonl", "log_path": null, "synthesis_doc": "PAVILION_WEEYUGA_v1.md", "cells": [ { "cell_id": "pavilion:weeyuga:qwen3.5:4b", "n_calls": 2, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-uncensored-iq1m", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:35b-a3b-iq2s", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q6k", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b-q4km", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:2b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:0.8b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3.5:9b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:14b", "n_calls": 1, "n_errors": 1, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:3b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:14b", "n_calls": 1, "n_errors": 1, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:8b", "n_calls": 1, "n_errors": 1, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen3:4b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5:3b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:1.5b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null }, { "cell_id": "pavilion:weeyuga:qwen2.5-coder:0.5b", "n_calls": 1, "n_errors": 0, "duration_ms_mean": null, "duration_ms_p50": null, "tokens_per_sec_mean": null, "tokens_per_sec_max": null } ], "n_calls_total": 17, "n_errors_total": 3, "started_at": "2026-04-28T21:03:46Z", "tags": [ "a3b", "iq2", "pavilion", "qwen2.5", "qwen3", "qwen3.5", "weeyuga" ], "status": "complete", "visibility": "draft" } ], "skipped": [ "ff1131ca-d021-4e06-8616-4b4cdb54e97e-reconstructed-from-log.jsonl (reconstructed duplicate)" ] }