current commit
261026.53 · Code pages render twice as fast: lines share one array, the page one builder
package main

import (
	"go/scanner"
	"go/token"
	"strings"

	"github.com/alecthomas/chroma/v2"
)

// goTokens lexes Go with the standard library's scanner, about 150 times
// faster than chroma's regular expressions. It types tokens as chroma's Go
// lexer does, so the colours stay the same.
func goTokens(code string) []chroma.Token {
	src := []byte(code)
	var s scanner.Scanner
	file := token.NewFileSet().AddFile("", -1, len(src))
	s.Init(file, src, nil, scanner.ScanComments)

	type lexeme struct {
		start, end int
		tok        token.Token
		lit        string
	}
	// scan returns the next lexeme. A name's type depends on the lexeme
	// after it, so the loop below looks one ahead.
	scan := func() (lexeme, bool) {
		for {
			pos, tok, lit := s.Scan()
			if tok == token.EOF {
				return lexeme{}, false
			}
			// The scanner inserts a semicolon at a line's end; it has no text.
			if tok == token.SEMICOLON && lit != ";" {
				continue
			}
			start := file.Offset(pos)
			text := lit
			if text == "" {
				text = tok.String()
			}
			// Comments and raw strings come without carriage returns, so
			// their end is found in the source, not from the literal's
			// length.
			end := min(start+len(text), len(src))
			if tok == token.COMMENT || tok == token.STRING && strings.HasPrefix(lit, "`") {
				end = rawEnd(code, start, tok)
			}
			return lexeme{start, end, tok, code[start:end]}, true
		}
	}

	// Go source runs about one token to three bytes, spaces included.
	out := make([]chroma.Token, 0, len(code)/3+16)
	last := 0
	for l, ok := scan(); ok; {
		next, more := scan()
		if l.start >= last {
			if l.start > last {
				out = append(out, chroma.Token{Type: chroma.Text, Value: code[last:l.start]})
			}
			paren := more && next.tok == token.LPAREN
			tight := paren && next.start == l.end
			out = append(out, chroma.Token{Type: goType(l.tok, l.lit, paren, tight), Value: l.lit})
			last = l.end
		}
		l, ok = next, more
	}
	if last < len(code) {
		out = append(out, chroma.Token{Type: chroma.Text, Value: code[last:]})
	}
	return out
}

// rawEnd finds the end of a comment or raw string that starts at start.
// It slices code rather than converting the source's bytes, which would
// copy the rest of the file for each comment.
func rawEnd(code string, start int, tok token.Token) int {
	rest := code[start:]
	var close string
	var from int
	switch {
	case tok == token.STRING:
		close, from = "`", 1
	case strings.HasPrefix(rest, "/*"):
		close, from = "*/", 2
	default:
		if i := strings.IndexByte(rest, '\n'); i >= 0 {
			return start + i
		}
		return len(code)
	}
	if i := strings.Index(rest[from:], close); i >= 0 {
		return start + from + i + len(close)
	}
	return len(code)
}

var goBuiltinFuncs = setOf("uint", "uint8", "uint16", "uint32", "uint64", "int", "int8", "int16", "int32", "int64",
	"float", "float32", "float64", "complex64", "complex128", "byte", "rune", "string", "bool", "error", "uintptr",
	"print", "println", "panic", "recover", "close", "complex", "real", "imag", "len", "cap", "append", "copy",
	"delete", "new", "make", "clear", "min", "max")

var goBuiltinTypes = setOf("uint", "uint8", "uint16", "uint32", "uint64", "int", "int8", "int16", "int32", "int64",
	"float", "float32", "float64", "complex64", "complex128", "byte", "rune", "string", "bool", "error", "uintptr", "any")

func setOf(words ...string) map[string]bool {
	m := make(map[string]bool, len(words))
	for _, w := range words {
		m[w] = true
	}
	return m
}

// goType follows chroma's Go rules: a name before ( is a function, or a
// builtin when the ( follows at once.
func goType(tok token.Token, lit string, paren, tight bool) chroma.TokenType {
	switch {
	case tok == token.COMMENT:
		if strings.HasPrefix(lit, "//") && len(lit) > 2 && !strings.ContainsRune(" \t\r\n", rune(lit[2])) {
			return chroma.CommentPreproc
		}
		return chroma.Comment
	case tok.IsKeyword():
		return chroma.Keyword
	case tok == token.IDENT:
		switch {
		case lit == "true" || lit == "false" || lit == "iota" || lit == "nil":
			return chroma.KeywordConstant
		case tight && goBuiltinFuncs[lit]:
			return chroma.NameBuiltin
		case goBuiltinTypes[lit]:
			return chroma.KeywordType
		case paren:
			return chroma.NameFunction
		}
		return chroma.NameOther
	case tok == token.CHAR:
		return chroma.LiteralStringChar
	case tok == token.STRING:
		return chroma.LiteralString
	case tok == token.FLOAT:
		return chroma.LiteralNumberFloat
	case tok == token.IMAG:
		return chroma.LiteralNumber
	case tok == token.INT:
		lower := strings.ToLower(lit)
		switch {
		case strings.HasPrefix(lower, "0x"):
			return chroma.LiteralNumberHex
		case strings.HasPrefix(lower, "0b"):
			return chroma.LiteralNumberBin
		case strings.HasPrefix(lower, "0o"), len(lit) > 1 && lit[0] == '0':
			return chroma.LiteralNumberOct
		}
		return chroma.LiteralNumberInteger
	case tok.IsOperator():
		return chroma.Operator
	}
	return chroma.Error
}