| { |
| "schema_version": 1, |
| "created_at": "2026-08-06T03:11:47.640376+00:00", |
| "definition": "First place means rank 1 by both fertility and effective bits per byte among the successfully loaded, revision-pinned public comparison set.", |
| "datasets": { |
| "public_korean": { |
| "id": "eduagarcia/multilingual_tokenizer_benchmark", |
| "revision": "74fa5907af0265f42c09e21379bf21847e066990", |
| "domains": [ |
| "finance", |
| "legal", |
| "lyrics", |
| "news", |
| "social", |
| "subtitles", |
| "web", |
| "wiki" |
| ], |
| "selection_role": "development evaluation", |
| "text_examples_redistributed": false, |
| "results": { |
| "korbyte-128k": { |
| "model_id": "dawncr0w/KorByte-128K", |
| "revision": null, |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2318164, |
| "fertility": 1.4462979761995376, |
| "characters_per_token": 2.6143340160575352, |
| "bytes_per_token": 5.4064143002824645, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 99566, |
| "ebpb": 3.07104942931184, |
| "throughput_mib_s": 17.796041101582244 |
| }, |
| "ranks": { |
| "fertility": 1, |
| "ebpb": 1 |
| } |
| }, |
| "ax-4.0": { |
| "model_id": "skt/A.X-4.0", |
| "revision": "a3393fd67bf09a34a85e66dfd2c2da61c4497b67", |
| "vocabulary_size": 102402, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2497463, |
| "fertility": 1.5581622708890421, |
| "characters_per_token": 2.4266445588983703, |
| "bytes_per_token": 5.018274545008274, |
| "exact_document_ratio": 0.9987365761212887, |
| "byte_fidelity": 0.9999967286246539, |
| "character_fidelity": 0.9999976899424218, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 71975, |
| "ebpb": 3.2153162310868675, |
| "throughput_mib_s": 17.108109016528726 |
| }, |
| "ranks": { |
| "fertility": 5, |
| "ebpb": 5 |
| } |
| }, |
| "trillion-7b": { |
| "model_id": "trillionlabs/Trillion-7B-preview", |
| "revision": "bc1650c5a954c7de0509d28c8dbc5bea560856e1", |
| "vocabulary_size": 128128, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2584960, |
| "fertility": 1.612751477702508, |
| "characters_per_token": 2.3445062979697946, |
| "bytes_per_token": 4.848413515102748, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 52099, |
| "ebpb": 3.2317722100425508, |
| "throughput_mib_s": 17.197271438330166 |
| }, |
| "ranks": { |
| "fertility": 7, |
| "ebpb": 6 |
| } |
| }, |
| "midm-2.0": { |
| "model_id": "K-intelligence/Midm-2.0-Base-Instruct", |
| "revision": "35479c5fc9a18a5db7cc6dbadcf1db68db7beab0", |
| "vocabulary_size": 131384, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2570110, |
| "fertility": 1.6034865918072205, |
| "characters_per_token": 2.358052768169456, |
| "bytes_per_token": 4.876427468085024, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 68991, |
| "ebpb": 3.2962902979186577, |
| "throughput_mib_s": 17.83905852965348 |
| }, |
| "ranks": { |
| "fertility": 6, |
| "ebpb": 7 |
| } |
| }, |
| "k-exaone": { |
| "model_id": "LGAI-EXAONE/K-EXAONE-236B-A23B", |
| "revision": "61e6d578eb102b578e5704e2916ac841df9eca0a", |
| "vocabulary_size": 153600, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2449553, |
| "fertility": 1.5282713157884886, |
| "characters_per_token": 2.474106500247188, |
| "bytes_per_token": 5.116425323314091, |
| "exact_document_ratio": 0.9987365761212887, |
| "byte_fidelity": 0.9999967286246539, |
| "character_fidelity": 0.9999976899424218, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 65457, |
| "ebpb": 3.1268693967808563, |
| "throughput_mib_s": 28.686802012815132 |
| }, |
| "ranks": { |
| "fertility": 2, |
| "ebpb": 3 |
| } |
| }, |
| "solar-pro3": { |
| "model_id": "upstage/solar-pro3-tokenizer", |
| "revision": "c41b71f0519c580610cb0fd2af4ed2b23cad544f", |
| "vocabulary_size": 196608, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2453228, |
| "fertility": 1.5305641410858073, |
| "characters_per_token": 2.4704002237052567, |
| "bytes_per_token": 5.108760783751041, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 70822, |
| "ebpb": 3.1537804660058923, |
| "throughput_mib_s": 18.1775053579418 |
| }, |
| "ranks": { |
| "fertility": 3, |
| "ebpb": 4 |
| } |
| }, |
| "thunder": { |
| "model_id": "thunder-research-group/Llama-Thunder-LLM-8B", |
| "revision": "ecfd6c909944939ae42982f3d87cf36898da308b", |
| "error": "Exception: data did not match any variant of untagged enum ModelUntagged at line 770599 column 1" |
| }, |
| "kanana-2": { |
| "model_id": "kakaocorp/kanana-2-3b-base", |
| "revision": "9fa2a678c379700868d72685a22ea8ed0263ef33", |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2482924, |
| "fertility": 1.5490914172842218, |
| "characters_per_token": 2.4408540092246076, |
| "bytes_per_token": 5.04765953367884, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 55554, |
| "ebpb": 3.122556714986635, |
| "throughput_mib_s": 24.74148599340686 |
| }, |
| "ranks": { |
| "fertility": 4, |
| "ebpb": 2 |
| } |
| }, |
| "hyperclovax": { |
| "model_id": "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B", |
| "revision": "a6cdfd3464d1b767259cad23e164eaf39d3e3960", |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 9498, |
| "words": 1602826, |
| "characters": 6060455, |
| "utf8_bytes": 12532955, |
| "tokens": 2908889, |
| "fertility": 1.8148501459297515, |
| "characters_per_token": 2.08342600903644, |
| "bytes_per_token": 4.308502318239025, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 53073, |
| "ebpb": 3.642957417899077, |
| "throughput_mib_s": 17.31447378801254 |
| }, |
| "ranks": { |
| "fertility": 8, |
| "ebpb": 8 |
| } |
| } |
| } |
| }, |
| "kmmlu_test": { |
| "id": "HAERAE-HUB/KMMLU", |
| "revision": "d61b3f19e552c576bf5960dd24289763edc36a88", |
| "subjects": 45, |
| "selection_role": "frozen post-selection audit; no algorithm changes followed", |
| "text_examples_redistributed": false, |
| "results": { |
| "korbyte-128k": { |
| "model_id": "dawncr0w/KorByte-128K", |
| "revision": null, |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2136460, |
| "fertility": 1.9363674100499033, |
| "characters_per_token": 2.24028345955459, |
| "bytes_per_token": 4.803871825355962, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 58038, |
| "ebpb": 3.2941574545664487, |
| "throughput_mib_s": 15.39604399507986 |
| }, |
| "ranks": { |
| "fertility": 1, |
| "ebpb": 1 |
| } |
| }, |
| "ax-4.0": { |
| "model_id": "skt/A.X-4.0", |
| "revision": "a3393fd67bf09a34a85e66dfd2c2da61c4497b67", |
| "vocabulary_size": 102402, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2450486, |
| "fertility": 2.220982948046557, |
| "characters_per_token": 1.953194590787297, |
| "bytes_per_token": 4.1882630629189475, |
| "exact_document_ratio": 0.995575221238938, |
| "byte_fidelity": 0.9999150369082789, |
| "character_fidelity": 0.9999389922352995, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 43443, |
| "ebpb": 3.6792538251829634, |
| "throughput_mib_s": 14.38818129811692 |
| }, |
| "ranks": { |
| "fertility": 6, |
| "ebpb": 7 |
| } |
| }, |
| "trillion-7b": { |
| "model_id": "trillionlabs/Trillion-7B-preview", |
| "revision": "bc1650c5a954c7de0509d28c8dbc5bea560856e1", |
| "vocabulary_size": 128128, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2456313, |
| "fertility": 2.226264213737635, |
| "characters_per_token": 1.9485611157861396, |
| "bytes_per_token": 4.178327436283568, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 34265, |
| "ebpb": 3.6053775814055884, |
| "throughput_mib_s": 14.986220352651252 |
| }, |
| "ranks": { |
| "fertility": 7, |
| "ebpb": 5 |
| } |
| }, |
| "midm-2.0": { |
| "model_id": "K-intelligence/Midm-2.0-Base-Instruct", |
| "revision": "35479c5fc9a18a5db7cc6dbadcf1db68db7beab0", |
| "vocabulary_size": 131384, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2450350, |
| "fertility": 2.220859685281157, |
| "characters_per_token": 1.953302997530965, |
| "bytes_per_token": 4.188495521048013, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 42024, |
| "ebpb": 3.666931429668025, |
| "throughput_mib_s": 15.025529566263533 |
| }, |
| "ranks": { |
| "fertility": 5, |
| "ebpb": 6 |
| } |
| }, |
| "k-exaone": { |
| "model_id": "LGAI-EXAONE/K-EXAONE-236B-A23B", |
| "revision": "61e6d578eb102b578e5704e2916ac841df9eca0a", |
| "vocabulary_size": 153600, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2367509, |
| "fertility": 2.145777253306796, |
| "characters_per_token": 2.021650604073733, |
| "bytes_per_token": 4.335054270121043, |
| "exact_document_ratio": 0.995575221238938, |
| "byte_fidelity": 0.9999150369082789, |
| "character_fidelity": 0.9999389922352995, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 38758, |
| "ebpb": 3.516715619432391, |
| "throughput_mib_s": 19.086997350623356 |
| }, |
| "ranks": { |
| "fertility": 3, |
| "ebpb": 4 |
| } |
| }, |
| "solar-pro3": { |
| "model_id": "upstage/solar-pro3-tokenizer", |
| "revision": "c41b71f0519c580610cb0fd2af4ed2b23cad544f", |
| "vocabulary_size": 196608, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2315383, |
| "fertility": 2.0985331730917385, |
| "characters_per_token": 2.0671638342339045, |
| "bytes_per_token": 4.432648939721852, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 48897, |
| "ebpb": 3.5142549173071034, |
| "throughput_mib_s": 15.146371887703577 |
| }, |
| "ranks": { |
| "fertility": 2, |
| "ebpb": 2 |
| } |
| }, |
| "thunder": { |
| "model_id": "thunder-research-group/Llama-Thunder-LLM-8B", |
| "revision": "ecfd6c909944939ae42982f3d87cf36898da308b", |
| "error": "Exception: data did not match any variant of untagged enum ModelUntagged at line 770599 column 1" |
| }, |
| "kanana-2": { |
| "model_id": "kakaocorp/kanana-2-3b-base", |
| "revision": "9fa2a678c379700868d72685a22ea8ed0263ef33", |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2385998, |
| "fertility": 2.1625346449941723, |
| "characters_per_token": 2.0059849170032833, |
| "bytes_per_token": 4.301462113547455, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 35675, |
| "ebpb": 3.51569429418471, |
| "throughput_mib_s": 17.819925688602993 |
| }, |
| "ranks": { |
| "fertility": 4, |
| "ebpb": 3 |
| } |
| }, |
| "hyperclovax": { |
| "model_id": "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B", |
| "revision": "a6cdfd3464d1b767259cad23e164eaf39d3e3960", |
| "vocabulary_size": 128256, |
| "metrics": { |
| "documents": 35030, |
| "words": 1103334, |
| "characters": 4786276, |
| "utf8_bytes": 10263280, |
| "tokens": 2775635, |
| "fertility": 2.5156797488339886, |
| "characters_per_token": 1.7243895541020342, |
| "bytes_per_token": 3.6976331542151617, |
| "exact_document_ratio": 1.0, |
| "byte_fidelity": 1.0, |
| "character_fidelity": 1.0, |
| "unknown_token_ratio": 0.0, |
| "observed_vocabulary": 31612, |
| "ebpb": 4.042635942884232, |
| "throughput_mib_s": 14.72578371433208 |
| }, |
| "ranks": { |
| "fertility": 8, |
| "ebpb": 8 |
| } |
| } |
| } |
| } |
| }, |
| "first_place_gate_passed": true, |
| "comparison_coverage_gate_passed": true, |
| "required_successful_systems": [ |
| "ax-4.0", |
| "hyperclovax", |
| "k-exaone", |
| "kanana-2", |
| "korbyte-128k", |
| "midm-2.0", |
| "solar-pro3", |
| "trillion-7b" |
| ], |
| "limitations": [ |
| "Intrinsic tokenization metrics do not establish downstream model quality.", |
| "The comparison set is broad and current but cannot prove a universal ranking.", |
| "Thunder's public tokenizer uses an unsupported custom `Beta` model and is recorded as unavailable." |
| ], |
| "environment": { |
| "platform": "macOS-26.5-arm64-arm-64bit", |
| "processor": "arm", |
| "python": "3.11.15", |
| "logical_cpu_count": 15, |
| "physical_memory_bytes": 51539607552, |
| "packages": { |
| "huggingface-hub": "1.26.0", |
| "rapidfuzz": "3.14.5", |
| "tokenizers": "0.22.2", |
| "transformers": "5.14.1" |
| } |
| } |
| } |
|
|
