doc/code-pages.md

@@ -1,6 +1,6 @@ # Code pages -Code in a fenced block with a known language is highlighted by chroma, as are code pages, whose language follows the file name. Go is lexed by Go's own scanner from the standard library instead, about 150 times faster, with the same colours. Python, JavaScript and shell have lexers of www's own, which follow chroma's rules and give every byte the class chroma gives it; chroma took 180 ms to lex argparse.py, 2,633 lines, and they take 2 ms. A whole page of it renders in about 5 ms, and jquery.js, 10,907 lines, in 28 ms instead of 400. Other languages go through chroma, at about 70 µs a line. The server keeps each coloured page by its content's hash, within 16 MB, so a later view takes a few milliseconds.+Code in a fenced block with a known language is highlighted by chroma, as are code pages, whose language follows the file name. Go is lexed by Go's own scanner from the standard library instead, about 150 times faster, with the same colours. Python, JavaScript and shell have lexers of www's own, which follow chroma's rules and give every byte the class chroma gives it; chroma took 180 ms to lex argparse.py, 2,633 lines, and they take under 2 ms. A whole page of it renders in about 3 ms, and jquery.js, 10,907 lines, in about 20 ms instead of 400. A Go page of 1,400 lines takes 2 ms. Other languages go through chroma, at about 70 µs a line. The server keeps each coloured page by its content's hash, within 16 MB, so a later view takes a few milliseconds.  A code page anchors lines as GitHub does, so `main.go#L42` lands on line 42 and highlights it. Clicking a line's number highlights it and puts its address in the address bar; clicking the number again clears both. A range such as `main.go#L10-L20` highlights those lines. 

golex.go

@@ -22,46 +22,49 @@ func goTokens(code string) []chroma.Token { 		tok        token.Token 		lit        string 	}-	var lexemes []lexeme-	for {-		pos, tok, lit := s.Scan()-		if tok == token.EOF {-			break+	// scan returns the next lexeme. A name's type depends on the lexeme+	// after it, so the loop below looks one ahead.+	scan := func() (lexeme, bool) {+		for {+			pos, tok, lit := s.Scan()+			if tok == token.EOF {+				return lexeme{}, false+			}+			// The scanner inserts a semicolon at a line's end; it has no text.+			if tok == token.SEMICOLON && lit != ";" {+				continue+			}+			start := file.Offset(pos)+			text := lit+			if text == "" {+				text = tok.String()+			}+			// Comments and raw strings come without carriage returns, so+			// their end is found in the source, not from the literal's+			// length.+			end := min(start+len(text), len(src))+			if tok == token.COMMENT || tok == token.STRING && strings.HasPrefix(lit, "`") {+				end = rawEnd(code, start, tok)+			}+			return lexeme{start, end, tok, code[start:end]}, true 		}-		// The scanner inserts a semicolon at a line's end; it has no text.-		if tok == token.SEMICOLON && lit != ";" {-			continue-		}-		start := file.Offset(pos)-		text := lit-		if text == "" {-			text = tok.String()-		}-		// Comments and raw strings come without carriage returns, so their-		// end is found in the source, not from the literal's length.-		end := min(start+len(text), len(src))-		if tok == token.COMMENT || tok == token.STRING && strings.HasPrefix(lit, "`") {-			end = rawEnd(code, start, tok)-		}-		lexemes = append(lexemes, lexeme{start, end, tok, code[start:end]}) 	} -	var out []chroma.Token+	// Go source runs about one token to three bytes, spaces included.+	out := make([]chroma.Token, 0, len(code)/3+16) 	last := 0-	for i, l := range lexemes {-		if l.start < last {-			continue-		}-		if l.start > last {-			out = append(out, chroma.Token{Type: chroma.Text, Value: code[last:l.start]})-		}-		paren, tight := false, false-		if i+1 < len(lexemes) && lexemes[i+1].tok == token.LPAREN {-			paren = true-			tight = lexemes[i+1].start == l.end+	for l, ok := scan(); ok; {+		next, more := scan()+		if l.start >= last {+			if l.start > last {+				out = append(out, chroma.Token{Type: chroma.Text, Value: code[last:l.start]})+			}+			paren := more && next.tok == token.LPAREN+			tight := paren && next.start == l.end+			out = append(out, chroma.Token{Type: goType(l.tok, l.lit, paren, tight), Value: l.lit})+			last = l.end 		}-		out = append(out, chroma.Token{Type: goType(l.tok, l.lit, paren, tight), Value: l.lit})-		last = l.end+		l, ok = next, more 	} 	if last < len(code) { 		out = append(out, chroma.Token{Type: chroma.Text, Value: code[last:]})

jslex.go

@@ -13,7 +13,7 @@ import ( // an opening bracket starts a regular expression, and one that does not // close marks the rest of its line as an error. func jsTokens(code string) []chroma.Token {-	j := &jsLex{lex: lex{src: code}}+	j := &jsLex{lex: newLex(code)} 	i := 0 	if strings.HasPrefix(code, "#!") { 		if k := strings.IndexByte(code, '\n'); k >= 0 && (strings.HasPrefix(code, "#!/") || strings.HasPrefix(code, "#! /")) {

lex.go

@@ -39,6 +39,12 @@ type span struct { 	from, to int } +// newLex makes room for the spans up front: JavaScript runs about one+// span to four bytes, Python and shell one to six or seven.+func newLex(code string) lex {+	return lex{src: code, spans: make([]span, 0, len(code)/4+16)}+}+ func (l *lex) emit(t chroma.TokenType, from, to int) { 	if to <= from { 		return

pylex.go

@@ -12,7 +12,7 @@ import ( // quirks included: a keyword stays one after a dot, 1j is a number and a // name, and a docstring is an ordinary string. func pyTokens(code string) []chroma.Token {-	p := &pyLex{lex{src: code}}+	p := &pyLex{newLex(code)} 	for i := 0; i < len(code); { 		i = p.step(i, false) 	}

render.go

@@ -257,10 +257,17 @@ func warmLexers() { // Escaping keeps file content from forging an id. Scoped lines, which // must be numbered, draw each block of code as a box behind it. func highlight(w io.Writer, lexer chroma.Lexer, code string, numbered, scoped bool) {+	// Every write goes to one builder: a builder of its own, or w when w is+	// one, as for a code page, so the page is never copied whole.+	b, ok := w.(*strings.Builder)+	if !ok {+		b = new(strings.Builder)+		defer func() { io.WriteString(w, b.String()) }()+	} 	lang := langName(lexer)-	lines := chroma.SplitTokensIntoLines(tokenise(lexer, code))+	lines := splitLines(tokenise(lexer, code)) 	// A final newline ends the last line; it does not start another.-	if len(lines) > 0 && lineText(lines[len(lines)-1]) == "" {+	if len(lines) > 0 && blank(lines[len(lines)-1]) { 		lines = lines[:len(lines)-1] 	} 	var scopes []scope@@ -274,27 +281,97 @@ func highlight(w io.Writer, lexer chroma.Lexer, code string, numbered, scoped bo 	if len(scopes) > 0 { 		pre += " scopes" 	}-	io.WriteString(w, `<div class="sourceCode"><pre class="`+strings.Join(strings.Fields(pre), " ")+`"><code class="`+strings.TrimSpace("sourceCode "+lang)+`">`)+	b.Grow(pageSize(code, lines, len(scopes), numbered))+	b.WriteString(`<div class="sourceCode"><pre class="` + strings.Join(strings.Fields(pre), " ") + `"><code class="` + strings.TrimSpace("sourceCode "+lang) + `">`)+	var num [20]byte 	for i, line := range lines { 		if i > 0 {-			io.WriteString(w, "\n")+			b.WriteByte('\n') 		} 		if numbered {-			id := "L" + strconv.Itoa(i+1)-			io.WriteString(w, `<span id="`+id+`"><a href="#`+id+`"></a>`)+			id := strconv.AppendInt(num[:0], int64(i+1), 10)+			b.WriteString(`<span id="L`)+			b.Write(id)+			b.WriteString(`"><a href="#L`)+			b.Write(id)+			b.WriteString(`"></a>`) 			n := 0 			for n < len(scopes) && scopes[n].first == i { 				n++ 			}-			writeScopes(w, scopes[:n])+			writeScopes(b, scopes[:n]) 			scopes = scopes[n:] 		} else {-			io.WriteString(w, "<span>")+			b.WriteString("<span>")+		}+		writeLine(b, line)+		b.WriteString("</span>")+	}+	b.WriteString("</code></pre></div>")+}++// pageSize guesses the length of highlight's output, so the builder grows+// once. Each token is weighed at 16 bytes, as if about half carried a+// span of their own; on this repo's code and Debian's jquery.js,+// bash_completion, argparse.py and stdio.h the guess is 7 to 41% over.+// Slack stays in a cached page unseen, and a builder that doubles leaves+// up to 100%.+func pageSize(code string, lines [][]chroma.Token, scopes int, numbered bool) int {+	tokens := 0+	for _, line := range lines {+		tokens += len(line)+	}+	perLine := len("\n<span></span>")+	if numbered {+		perLine = len(`\n<span id="L1234"><a href="#L1234"></a></span>`)+	}+	box := len(`<i class="scope leaf shade" style="--c:12;--w:34;--d:3"></i>`)+	return len(code) + len(code)/8 + tokens*16 + len(lines)*perLine + scopes*box+}++// splitLines splits tokens into lines as chroma.SplitTokensIntoLines+// does: a token that spans lines becomes one per line, each ending in its+// newline. Chroma allocates each line on its own, which took a third of+// the page writer; here one array backs every line.+func splitLines(tokens []chroma.Token) [][]chroma.Token {+	n, rows := len(tokens), 1+	for _, tok := range tokens {+		c := strings.Count(tok.Value, "\n")+		n += c+		rows += c+	}+	all := make([]chroma.Token, 0, n)+	lines := make([][]chroma.Token, 0, rows)+	start := 0+tokens:+	for _, tok := range tokens {+		for {+			i := strings.IndexByte(tok.Value, '\n')+			if i < 0 {+				break+			}+			head := tok+			head.Value = tok.Value[:i+1]+			all = append(all, head)+			lines = append(lines, all[start:len(all):len(all)])+			start = len(all)+			// An empty tail starts nothing.+			if tok.Value = tok.Value[i+1:]; tok.Value == "" {+				continue tokens+			}+		}+		all = append(all, tok)+	}+	if start < len(all) {+		lines = append(lines, all[start:len(all):len(all)])+	}+	// Chroma drops a last line that holds only an empty token.+	if len(lines) > 0 {+		if last := lines[len(lines)-1]; len(last) == 1 && last[0].Value == "" {+			lines = lines[:len(lines)-1] 		}-		writeLine(w, line)-		io.WriteString(w, "</span>") 	}-	io.WriteString(w, "</code></pre></div>")+	return lines }  // codeCache keeps highlighted code pages by lexer and content hash. Chroma@@ -318,41 +395,66 @@ func highlightPage(lexer chroma.Lexer, b []byte, scoped bool) string { }  // writeLine writes one line's tokens, merging neighbours of one class.-func writeLine(w io.Writer, line []chroma.Token) {-	var run strings.Builder+func writeLine(b *strings.Builder, line []chroma.Token) { 	class := ""-	flush := func() {-		if run.Len() == 0 {-			return-		}-		text := template.HTMLEscapeString(run.String())-		if class == "" {-			io.WriteString(w, text)-		} else {-			io.WriteString(w, `<span class="`+class+`">`+text+`</span>`)-		}-		run.Reset()-	} 	for _, tok := range line { 		v := strings.TrimSuffix(tok.Value, "\n") 		if v == "" { 			continue 		} 		if c := pandocClass(tok.Type); c != class {-			flush()+			if class != "" {+				b.WriteString("</span>")+			}+			if c != "" {+				b.WriteString(`<span class="` + c + `">`)+			} 			class = c 		}-		run.WriteString(v)+		escapeHTML(b, v)+	}+	if class != "" {+		b.WriteString("</span>")+	}+}++// escapeHTML writes s escaped as template.HTMLEscapeString escapes it,+// without making a string of its own.+func escapeHTML(b *strings.Builder, s string) {+	last := 0+	for i := 0; i < len(s); i++ {+		var esc string+		switch s[i] {+		case '"':+			esc = "&#34;"+		case '\'':+			esc = "&#39;"+		case '&':+			esc = "&amp;"+		case '<':+			esc = "&lt;"+		case '>':+			esc = "&gt;"+		case 0:+			esc = "\uFFFD"+		default:+			continue+		}+		b.WriteString(s[last:i])+		b.WriteString(esc)+		last = i + 1 	}-	flush()+	b.WriteString(s[last:]) } -func lineText(line []chroma.Token) string {-	var s strings.Builder+// blank reports whether a line holds no text.+func blank(line []chroma.Token) bool { 	for _, tok := range line {-		s.WriteString(tok.Value)+		if tok.Value != "" {+			return false+		} 	}-	return s.String()+	return true }  // langName is the lexer's class on the code element: its first alias, eg

scope.go

@@ -1,7 +1,6 @@ package main  import (-	"io" 	"slices" 	"strconv" 	"strings"@@ -262,19 +261,26 @@ func endsWithColon(line []chroma.Token) bool { // positioned on its own, so it adds no text and moves none; the // stylesheet turns the cells and rows into lengths. Boxes at odd depths // are shaded, so shade and page alternate inward.-func writeScopes(w io.Writer, scopes []scope) {+func writeScopes(b *strings.Builder, scopes []scope) {+	var num [20]byte+	cell := func(name string, n int) {+		b.WriteString(name)+		b.Write(strconv.AppendInt(num[:0], int64(n), 10))+	} 	for _, s := range scopes {-		class := "scope"+		b.WriteString(`<i class="scope`) 		if s.leaf > 0 {-			class += " leaf"+			b.WriteString(" leaf") 		} 		if s.depth%2 == 1 {-			class += " shade"+			b.WriteString(" shade") 		}-		io.WriteString(w, `<i class="`+class+`" style="--c:`+strconv.Itoa(s.left)+`;--w:`+strconv.Itoa(s.right-s.left))+		cell(`" style="--c:`, s.left)+		cell(`;--w:`, s.right-s.left) 		if s.leaf == 0 {-			io.WriteString(w, `;--r:`+strconv.Itoa(s.last-s.first+1))+			cell(`;--r:`, s.last-s.first+1) 		}-		io.WriteString(w, `;--d:`+strconv.Itoa(s.depth)+`"></i>`)+		cell(`;--d:`, s.depth)+		b.WriteString(`"></i>`) 	} }

scope_test.go

@@ -4,12 +4,10 @@ import ( 	"reflect" 	"strings" 	"testing"--	"github.com/alecthomas/chroma/v2" )  func scopesOf(name, code string) []scope {-	return findScopes(chroma.SplitTokensIntoLines(tokenise(codeLexer(name), code)), langName(codeLexer(name)) == "python")+	return findScopes(splitLines(tokenise(codeLexer(name), code)), langName(codeLexer(name)) == "python") }  // A block that spans lines is a scope, and brackets or a string on one

shlex.go

@@ -11,7 +11,7 @@ import ( // included: a keyword needs a word boundary on both sides, a builtin only // a space, ) or ` after it, and a number only a space or the line's end. func shTokens(code string) []chroma.Token {-	l := &shLex{lex: lex{src: code}}+	l := &shLex{lex: newLex(code)} 	i := 0 	if strings.HasPrefix(code, "#!") { 		if k := strings.IndexByte(code, '\n'); k > 2 {