Meet Gigatoken: A Rust BPE Tokenizer that Encodes Textual content at 24.53 GB/s, as much as 989x Quicker than HuggingFace Tokenizers

‘;
}
operate sprint(){ return ‘—‘; }

operate renderBench(key)>. ‘
+ ‘Dimmed bars are SentencePiece-based tokenizers, that are much less optimized in gigatoken. ‘
+ ‘tiktoken rows are crammed solely the place official assist exists.

‘;
return h;

operate renderPerf(){
var max = 1049, h=”

”
+ ‘Single-threaded GPT-2 pretokenizer throughput on 100 MB of OpenWebText, Apple Silicon, cargo bench with lto = “fats”. ‘
+ ‘A flowery-regex baseline runs the identical pretokenizer at ~47 MiB/s.

‘;
PERF.forEach(operate(s){
h += ‘

‘;
});
h += ‘

Internet consequence: 2.27× over the winnow + NEON baseline, 22.3× over the regex implementation.

‘;
return h;
}

operate fmt(sec){
if(sec < 1) return (sec*1000).toFixed(0)+’ ms’;
if(sec < 90) return sec.toFixed(1)+’ s’;
if(sec < 5400) return (sec/60).toFixed(1)+’ min’;
if(sec < 172800) return (sec/3600).toFixed(1)+’ hours’;
return (sec/86400).toFixed(1)+’ days’;
}

operate renderCalc(){
var opts=””;
Object.keys(DATA).forEach(operate(okay){
opts += ‘‘+DATA[k].label+’‘;
});
return ‘

‘
+ ‘

Linear extrapolation from the measured throughput above. It assumes the reported price holds ‘
+ ‘for the entire corpus, single machine, no I/O ceiling. Actual pipelines are often storage-bound earlier than they’re tokenizer-bound. ‘
+ ‘For scale: the gigatoken README notes that at EPYC charges you might tokenize all of Frequent Crawl — ‘
+ ‘roughly 130 trillion tokens — in slightly below 6.5 hours.

‘;
}

operate calcUpdate(){
var cpu = R.querySelector(‘#gt-cpu’).worth;
var idx = parseInt(R.querySelector(‘#gt-tok’).worth,10);
var gb = parseInt(R.querySelector(‘#gt-sz’).worth,10);
R.querySelector(‘#gt-szv’).textContent = gb.toLocaleString();
var row = DATA[cpu].rows[idx];
var tg = gb / row[1];
var h=”

‘;
if(row[2] !== null){
var th = (gb*1000) / row[2];
h += ‘

‘;
h += ‘

‘;
} else {
h += ‘

‘;
}
if(row[3] !== null){
var tt = (gb*1000) / row[3];
h += ‘

‘;
}
R.querySelector(‘#gt-res’).innerHTML = h;
}

operate fillTok(){
var cpu = R.querySelector(‘#gt-cpu’).worth, sel = R.querySelector(‘#gt-tok’), h=””;
DATA[cpu].rows.forEach(operate(r,i){ h += ‘‘+r[0]+’‘; });
sel.innerHTML = h;
}

var USAGE =
‘

Set up

‘
+ ‘

pip set up gigatoken

‘
+ ‘

Compatibility mode — drop-in, roughly 200–300× per the creator

‘
+ ‘

import gigatoken as gtnn'
  + '# Minimal change from current HuggingFace tokenizers utilizationn'
  + 'hf_tokenizer = ...ntokenizer = gt.Tokenizer(hf_tokenizer).as_hf()nn'
  + 'tokens = tokenizer.encode_batch(["This is a test string", "And here is another"])nn'
  + '# OR with tiktokenntiktokenizer = ...ntokenizer = gt.Tokenizer(tiktokenizer).as_tiktoken()

‘
+ ‘

Gigatoken API — quickest path, Rust reads the information instantly

‘
+ ‘

import gigatoken as gtnn'
  + 'tokenizer = gt.Tokenizer("Qwen/Qwen3-8B")  # accepts HF mannequin namesn'
  + 'file_source = gt.TextFileSource(["owt_train.txt"], separator=b"<|endoftext|>")n'
  + 'tokens = tokenizer.encode_files(file_source)

‘
+ ‘

Verify your personal tokenizer with out putting in something

‘
+ ‘

uvx --with tokenizers gigatoken bench 'openai-community/gpt2' owt_train.txt n'
  + '    --validate --doc-separator "<|endoftext|>"

‘
+ ‘

The benchmark subcommand validates output in opposition to HuggingFace tokenizers and prints a timing comparability. ‘
+ ‘On macOS, run it twice: the primary run triggers a safety scan that slows the Rust code.

‘;

var LIMITS =
‘

Identified points, acknowledged by the creator

‘
+ ‘
• WordPiece shouldn’t be supported but. BERT-family WordPiece fashions are out of scope for now.

‘
+ ‘

• SentencePiece is barely partly optimized. Gemma, Mistral, CodeLlama and Llama 2 vocabularies land within the 7–22× band, not the 1,000× band.

‘
+ ‘

• File sinks should not applied within the Gigatoken API.

‘
+ ‘

• Python iteration makes use of ABI3, which is slower than version-specific CPython APIs. Early experiments present a 2× achieve for overhead-bound instances.

‘
+ ‘

• Home windows is barely examined. The creator recommends WSL for now.

‘
+ ‘

• Compatibility mode prices efficiency. Actual output parity with HuggingFace tokenizers is preserved, however not the complete speedup.

‘
+ ‘

Acknowledged AI use

‘
+ ‘
• The creator states many of the code was written by hand, and that AI assisted with the user-facing API, compatibility widening, ‘
  + ‘porting SIMD methods between AVX512 / AVX2 / NEON, the ultimate profiling levels and roughly the final 4× of efficiency, and refactoring.

‘
+ ‘

License and packaging

‘
+ ‘
• MIT licensed. Rust 66.2% / Python 33.3% by repository language share. Printed to PyPI as gigatoken.

‘
+ ‘

‘;

var CMDS = {
epyc:{line:’gigatoken bench –cpu epyc-9565’, get:operate(){return renderBench(‘epyc’);}},
m4:{line:’gigatoken bench –cpu m4-max’, get:operate(){return renderBench(‘m4′);}},
ryzen:{line:’gigatoken bench –cpu ryzen-9800x3d’, get:operate(){return renderBench(‘ryzen’);}},
perf:{line:’gigatoken perf –log pretokenizer’, get:renderPerf},
calc:{line:’gigatoken estimate –corpus’, get:renderCalc},
utilization:{line:’gigatoken set up –show-usage’, get:operate(){return USAGE;}},
limits:{line:’gigatoken limits’, get:operate(){return LIMITS;}}
};

operate present(v){
var c = CMDS[v]; if(!c) return;
LINE.textContent = c.line;
BODY.innerHTML = c.get();
R.querySelectorAll(‘.gt-cmd’).forEach(operate(b){
b.setAttribute(‘aria-selected’, b.dataset.v === v ? ‘true’ : ‘false’);
});
if(v === ‘calc’){
fillTok(); calcUpdate();
R.querySelector(‘#gt-cpu’).addEventListener(‘change’, operate(){ fillTok(); calcUpdate(); });
R.querySelector(‘#gt-tok’).addEventListener(‘change’, calcUpdate);
R.querySelector(‘#gt-sz’).addEventListener(‘enter’, calcUpdate);
}
resize();
}

R.querySelectorAll(‘.gt-cmd’).forEach(operate(b){
b.addEventListener(‘click on’, operate(){ present(b.dataset.v); });
});

operate resize(){
attempt{
if(window.dad or mum !== window){
window.dad or mum.postMessage({gigatokenHeight: R.offsetHeight + 40}, ‘*’);
}
}catch(e){}
}
window.addEventListener(‘resize’, resize);

present(‘epyc’);
})();