220904.1 · 3e922d532232ccb78f0b9b8d9a4387919466bda8
bin/rust: add 'find-invalid-utf8' program
I periodically like to look for files that contain invalid UTF-8, but have poor tooling to do it. Both 'iconv' and to an extent ripgrep can do it, but this program does it exactly how I want to do it: look at all files line by line, and print lines containing invalid UTF-8. And crucially, print invalid UTF-8 bytes as hexadecimal and colorize them.
reading declarations
bin/rust/find-invalid-utf8/.gitignore
@@ -0,0 +1 @@+/target
bin/rust/find-invalid-utf8/Cargo.lock
@@ -0,0 +1,115 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 3++[[package]]+name = "anyhow"+version = "1.0.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a26fa4d7e3f2eebadf743988fc8aec9fa9a9e82611acafd77c1462ed6262440a"++[[package]]+name = "bstr"+version = "1.0.0-pre.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2c7f5c92fd658e9b84052fd01ebea0ae21d444828fb9de3c28aa48b9cf6c9e0"+dependencies = [+ "lazy_static",+ "memchr",+ "regex-automata",+ "serde",+]++[[package]]+name = "find-invalid-utf8"+version = "0.1.0"+dependencies = [+ "anyhow",+ "bstr",+ "termcolor",+ "walkdir",+]++[[package]]+name = "lazy_static"+version = "1.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"++[[package]]+name = "memchr"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"++[[package]]+name = "regex-automata"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"++[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]++[[package]]+name = "serde"+version = "1.0.144"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"++[[package]]+name = "termcolor"+version = "1.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"+dependencies = [+ "winapi-util",+]++[[package]]+name = "walkdir"+version = "2.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"+dependencies = [+ "same-file",+ "winapi",+ "winapi-util",+]++[[package]]+name = "winapi"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"+dependencies = [+ "winapi-i686-pc-windows-gnu",+ "winapi-x86_64-pc-windows-gnu",+]++[[package]]+name = "winapi-i686-pc-windows-gnu"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"++[[package]]+name = "winapi-util"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"+dependencies = [+ "winapi",+]++[[package]]+name = "winapi-x86_64-pc-windows-gnu"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
bin/rust/find-invalid-utf8/Cargo.toml
@@ -0,0 +1,16 @@+[package]+name = "find-invalid-utf8"+version = "0.1.0"+edition = "2021"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]+anyhow = "1.0.63"+bstr = "1.0.0-pre.3"+termcolor = "1.1.3"+walkdir = "2.3.2"++[[bin]]+name = "find-invalid-utf8"+path = "main.rs"
bin/rust/find-invalid-utf8/main.rs
@@ -0,0 +1,89 @@+use std::{ffi::OsString, fs::File, io::BufRead, path::Path};++use anyhow::Context;+use bstr::{io::BufReadExt, ByteSlice};+use termcolor::{Color, ColorChoice, ColorSpec, WriteColor};++fn main() -> anyhow::Result<()> {+ let mut args: Vec<OsString> = std::env::args_os().skip(1).collect();+ let dir = match args.pop() {+ None => OsString::from("."),+ Some(dir) => {+ anyhow::ensure!(+ args.is_empty(),+ "Usage: find-invalid-utf8 [<dir>]"+ );+ dir+ }+ };+ // Basically just iterate over every line in every file in a directory+ // tree, and if a line contains invalid UTF-8, print it. Invalid UTF-8+ // bytes are colorized and printed in their hexadecimal format.+ //+ // As a heuristic, files containing a NUL byte are assumed to be binary+ // and are skipped.+ let wtr = termcolor::BufferWriter::stdout(ColorChoice::Auto);+ let mut buf = termcolor::Buffer::ansi();+ for result in walkdir::WalkDir::new(dir) {+ let dent = result?;+ if !dent.file_type().is_file() {+ continue;+ }++ let file = File::open(dent.path())+ .with_context(|| format!("{}", dent.path().display()))?;+ let rdr = std::io::BufReader::new(file);++ buf.clear();+ let is_binary = print_utf8_errors(dent.path(), rdr, &mut buf)+ .with_context(|| format!("{}", dent.path().display()))?;+ if !is_binary {+ wtr.print(&buf)?;+ }+ }+ Ok(())+}++/// Print lines that contain bytes that aren't valid UTF-8. Bytes that aren't+/// valid UTF-8 are printed in hexadecimal format.+///+/// This returns true if the reader is suspected of being a binary file. In+/// that case, printing to `wtr` is stopped and `true` is returned.+fn print_utf8_errors<R: BufRead, W: WriteColor>(+ path: &Path,+ mut rdr: R,+ mut wtr: W,+) -> anyhow::Result<bool> {+ let mut is_binary = false;+ let (mut lineno, mut offset) = (1, 0);+ rdr.for_byte_line_with_terminator(|line| {+ if line.find_byte(b'\x00').is_some() {+ is_binary = true;+ return Ok(false);+ }++ let (cur_lineno, cur_offset) = (lineno, offset);+ lineno += 1;+ offset += line.len();+ if line.is_utf8() {+ return Ok(true);+ }+ write!(wtr, "{}:{}:{}:", path.display(), cur_lineno, cur_offset)?;+ for chunk in line.utf8_chunks() {+ if !chunk.valid().is_empty() {+ write!(wtr, "{}", chunk.valid())?;+ }+ if !chunk.invalid().is_empty() {+ let mut color = ColorSpec::new();+ color.set_fg(Some(Color::Red)).set_bold(true);+ wtr.set_color(&color)?;+ for &b in chunk.invalid().iter() {+ write!(wtr, r"\x{:X}", b)?;+ }+ wtr.reset()?;+ }+ }+ Ok(true)+ })?;+ Ok(is_binary)+}
bin/rust/find-invalid-utf8/rustfmt.toml
@@ -0,0 +1,2 @@+max_width = 79+use_small_heuristics = "max"