240329.1 · a4c0621fc851705c2934dfd52c18fe1b04bd2b0f

bin/rust: add new 'freq' program

This program computes frequency distributions of bytes, codepoints, graphemes or words on arbitrary data.

I've frequently wanted a program like this over the years because it's useful to know how "common" something is in a corpus. In particular, since many of the optimizations in `regex` depend on this sort of thing, it's nice to get more visibility into what real data look like.

new file mode 100644@@ -0,0 +1,22 @@+[package]+name = "freq"+version = "0.1.0"+edition = "2021"++[dependencies]+anyhow = "1.0.68"+bstr = "1.9.1"+lexopt = "0.3.0"+log = "0.4.21"+memchr = "2.7.2"+regex = "1.10.4"+rustc-hash = "1.1.0"+tabwriter = "1.4.0"+walkdir = "2.5.0"++[[bin]]+name = "freq"+path = "main.rs"++[profile.release]+debug = true