@ 261101.7 · fetching
current
package main

import (
	"strings"

	"github.com/alecthomas/chroma/v2"
)

// shTokens lexes shell as chroma's Bash lexer does, in one pass. Its rules
// live in chroma's lexers/embedded/bash.xml; this follows them, quirks
// included: a keyword needs a word boundary on both sides, a builtin only
// a space, ) or ` after it, and a number only a space or the line's end.
func shTokens(code string) []chroma.Token {
	l := &shLex{lex: newLex(code)}
	i := 0
	if strings.HasPrefix(code, "#!") {
		if k := strings.IndexByte(code, '\n'); k > 2 {
			l.emit(chroma.CommentPreproc, 0, k+1)
			i = k + 1
		}
	}
	for i < len(code) {
		i = l.step(i, 0)
	}
	return l.tokens()
}

type shLex struct{ lex }

var (
	shKeywords = setOf("if", "fi", "else", "while", "do", "done", "for", "then", "return", "function", "case",
		"select", "continue", "until", "esac", "elif")
	shBuiltins = setOf("alias", "bg", "bind", "break", "builtin", "caller", "cd", "command", "compgen", "complete",
		"declare", "dirs", "disown", "echo", "enable", "eval", "exec", "exit", "export", "false", "fc", "fg",
		"getopts", "hash", "help", "history", "jobs", "kill", "let", "local", "logout", "popd", "printf", "pushd",
		"pwd", "read", "readonly", "set", "shift", "shopt", "source", "suspend", "test", "time", "times", "trap",
		"true", "type", "typeset", "ulimit", "umask", "unalias", "unset", "wait")
)

// The states that end on a closer: $( ... ), $(( ... )), ` ... ` and
// ${ ... }. In them step reports the closer by returning -1-i.
const (
	shRoot = iota
	shParen
	shMath
	shBacktick
	shCurly
)

func isASCIIWord(c byte) bool {
	return c == '_' || isDigit(c) || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
}

// wordBoundary reports whether \b holds at i.
func wordBoundary(s string, i int) bool {
	before, after := false, false
	if i > 0 {
		r, _ := lastRune(s[:i])
		before = isWord(r)
	}
	if i < len(s) {
		r, _ := runeAt(s, i)
		after = isWord(r)
	}
	return before != after
}

func lastRune(s string) (rune, int) {
	for k := len(s) - 1; k >= 0 && k >= len(s)-4; k-- {
		if s[k] < 0x80 || s[k] >= 0xC0 {
			return runeAt(s, k)
		}
	}
	return runeAt(s, len(s)-1)
}

// step lexes one token at i in state st and returns where the next begins.
func (l *shLex) step(i int, st int) int {
	s := l.src
	switch st {
	case shParen:
		if s[i] == ')' {
			return -1 - i
		}
	case shMath:
		if strings.HasPrefix(s[i:], "))") {
			return -1 - i
		}
		switch {
		case strings.HasPrefix(s[i:], "**"), strings.HasPrefix(s[i:], "||"):
			l.emit(chroma.Operator, i, i+1)
			return i + 1
		case strings.IndexByte("-+*/%^|&", s[i]) >= 0:
			l.emit(chroma.Operator, i, i+1)
			return i + 1
		case isDigit(s[i]):
			e := i
			for e < len(s) && isDigit(s[e]) {
				e++
			}
			if e < len(s) && s[e] == '#' {
				f := e + 1
				for f < len(s) && isDigit(s[f]) {
					f++
				}
				if f > e+1 {
					e = f
				} else if e+1 >= len(s) || s[e+1] != ' ' {
					e++
				}
			}
			l.emit(chroma.LiteralNumber, i, e)
			return e
		}
	case shBacktick:
		if s[i] == '`' {
			return -1 - i
		}
	case shCurly:
		switch {
		case s[i] == '}':
			return -1 - i
		case strings.HasPrefix(s[i:], ":-"):
			l.emit(chroma.Keyword, i, i+2)
			return i + 2
		}
		if r, _ := runeAt(s, i); isWord(r) {
			e := wordEnd(s, i, isWord)
			l.emit(chroma.NameVariable, i, e)
			return e
		}
		if e := strings.IndexAny(s[i:], "}:\"'`$\\"); e != 0 {
			if e < 0 {
				e = len(s) - i
			}
			l.emit(chroma.Punctuation, i, i+e)
			return i + e
		}
		if s[i] == ':' {
			l.emit(chroma.Punctuation, i, i+1)
			return i + 1
		}
	}
	return l.root(i)
}

// root lexes one token by chroma's root state: basic, then backticks,
// data and interp.
func (l *shLex) root(i int) int {
	s := l.src
	c := s[i]
	if r, _ := runeAt(s, i); isWord(r) && wordBoundary(s, i) {
		w := s[i:wordEnd(s, i, isWord)]
		e := i + len(w)
		if shKeywords[w] {
			// (\s*)\b: the spaces go only as far as a boundary follows.
			ws := wordEnd(s, e, isSpace)
			for ws > e && !wordBoundary(s, ws) {
				ws--
			}
			l.emit(chroma.Keyword, i, e)
			l.emit(chroma.Text, e, ws)
			return ws
		}
		if shBuiltins[w] && e < len(s) {
			if r, _ := runeAt(s, e); isSpace(r) || r == ')' || r == '`' {
				l.emit(chroma.NameBuiltin, i, e)
				return e
			}
		}
	}
	switch {
	case c == '#':
		e := lineEnd(s, i)
		l.emit(chroma.CommentSingle, i, e)
		return e
	case c == '\\' && i+1 < len(s):
		_, n := runeAt(s, i+1)
		l.emit(chroma.LiteralStringEscape, i, i+1+n)
		return i + 1 + n
	}
	if e := l.assignment(i); e > i {
		return e
	}
	switch {
	case strings.IndexByte("[]{}()=", c) >= 0:
		l.emit(chroma.Operator, i, i+1)
		return i + 1
	case strings.HasPrefix(s[i:], "<<<"):
		l.emit(chroma.Operator, i, i+3)
		return i + 3
	case strings.HasPrefix(s[i:], "<<"):
		if e := shHeredoc(s, i); e > i {
			l.emit(chroma.LiteralString, i, e)
			return e
		}
	case strings.HasPrefix(s[i:], "&&"), strings.HasPrefix(s[i:], "||"):
		l.emit(chroma.Operator, i, i+2)
		return i + 2
	case c == '`':
		l.emit(chroma.LiteralStringBacktick, i, i+1)
		return l.nested(i+1, shBacktick, chroma.LiteralStringBacktick, 1)
	}
	return l.data(i)
}

// assignment lexes (\b\w+)(\s*)(\+?=) at i, a variable being set.
func (l *shLex) assignment(i int) int {
	s := l.src
	if r, _ := runeAt(s, i); !isWord(r) || !wordBoundary(s, i) {
		return i
	}
	e := wordEnd(s, i, isWord)
	k := e
	for k < len(s) && (s[k] == ' ' || s[k] == '\t' || s[k] == '\n' || s[k] == '\r' || s[k] == '\f' || s[k] == '\v') {
		k++
	}
	op := k
	if op < len(s) && s[op] == '+' {
		op++
	}
	if op >= len(s) || s[op] != '=' {
		return i
	}
	l.emit(chroma.NameVariable, i, e)
	l.emit(chroma.Text, e, k)
	l.emit(chroma.Operator, k, op+1)
	return op + 1
}

// shHeredoc matches <<-?\s*('?)\\?(\w+)[\w\W]+?\2 at i: the heredoc runs
// to the next place its word appears, as chroma's pattern does.
func shHeredoc(s string, i int) int {
	k := i + 2
	if k < len(s) && s[k] == '-' {
		k++
	}
	k = wordEnd(s, k, isSpace)
	if k < len(s) && s[k] == '\'' {
		k++
	}
	if k < len(s) && s[k] == '\\' {
		k++
	}
	e := wordEnd(s, k, isWord)
	if e == k {
		return i
	}
	// \w+ backtracks: the shortest word whose next appearance comes
	// soonest wins, so try each length from the longest down, as the
	// regex engine does, and keep the first that matches.
	for w := e; w > k; w-- {
		word := s[k:w]
		if m := strings.Index(s[w+1:], word); m >= 0 && w+1 <= len(s) {
			return w + 1 + m + len(word)
		}
	}
	return i
}

// data lexes chroma's data and interp rules.
func (l *shLex) data(i int) int {
	s := l.src
	c := s[i]
	switch {
	case c == '"' || c == '$' && i+1 < len(s) && s[i+1] == '"':
		start := i
		if c == '$' {
			i++
		}
		if e := shPlainDouble(s, i); e > 0 {
			l.emit(chroma.LiteralStringDouble, start, e)
			return e
		}
		if c == '"' {
			l.emit(chroma.LiteralStringDouble, i, i+1)
			return l.double(i + 1)
		}
	case c == '$' && i+1 < len(s) && s[i+1] == '\'':
		if e := shDollarSingle(s, i+1); e > 0 {
			l.emit(chroma.LiteralStringSingle, i, e)
			return e
		}
	case c == '\'':
		if k := strings.IndexByte(s[i+1:], '\''); k >= 0 {
			l.emit(chroma.LiteralStringSingle, i, i+1+k+1)
			return i + 1 + k + 1
		}
	case c == ';' || c == '&' || c == '|':
		l.emit(chroma.Punctuation, i, i+1)
		return i + 1
	}
	if r, _ := runeAt(s, i); isSpace(r) {
		e := wordEnd(s, i, isSpace)
		l.emit(chroma.Text, i, e)
		return e
	}
	if isDigit(c) {
		e := i
		for e < len(s) && isDigit(s[e]) {
			e++
		}
		// \d+(?= |$): the digits may end early where a space follows.
		for k := e; k > i; k-- {
			if k == len(s) || s[k] == ' ' || s[k] == '\n' {
				l.emit(chroma.LiteralNumber, i, k)
				return k
			}
		}
	}
	if e := shTextEnd(s, i); e > i {
		l.emit(chroma.Text, i, e)
		return e
	}
	if c == '<' {
		l.emit(chroma.Text, i, i+1)
		return i + 1
	}
	if c == '$' {
		return l.interp(i)
	}
	_, n := runeAt(s, i)
	l.emit(chroma.Error, i, i+n)
	return i + n
}

// shTextEnd returns the end of [^=\s\[\]{}()$"'`\\<&|;]+ at i.
func shTextEnd(s string, i int) int {
	return wordEnd(s, i, func(r rune) bool {
		return !isSpace(r) && !strings.ContainsRune("=[]{}()$\"'`\\<&|;", r)
	})
}

// shPlainDouble matches \$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*" at the quote i:
// a double-quoted string with no $ in it. It returns the end, or 0.
func shPlainDouble(s string, i int) int {
	for k := i + 1; k < len(s); k++ {
		switch s[k] {
		case '\\':
			k++
		case '$':
			return 0
		case '"':
			return k + 1
		}
	}
	return 0
}

// shDollarSingle matches \$'(\\\\|\\[0-7]+|\\.|[^'\\])*' from the quote i.
func shDollarSingle(s string, i int) int {
	for k := i + 1; k < len(s); k++ {
		switch s[k] {
		case '\\':
			k++
		case '\'':
			return k + 1
		}
	}
	return 0
}

// double lexes the rest of a double-quoted string that holds a $.
func (l *shLex) double(i int) int {
	s := l.src
	for i < len(s) {
		switch {
		case s[i] == '"':
			l.emit(chroma.LiteralStringDouble, i, i+1)
			return i + 1
		case s[i] == '$':
			i = l.interp(i)
		default:
			e := i
			for e < len(s) && s[e] != '"' && s[e] != '$' {
				if s[e] == '\\' && e+1 < len(s) {
					e++
				}
				e++
			}
			l.emit(chroma.LiteralStringDouble, i, e)
			i = e
		}
	}
	return i
}

// interp lexes a $ form at i.
func (l *shLex) interp(i int) int {
	s := l.src
	switch {
	case strings.HasPrefix(s[i:], "$(("):
		l.emit(chroma.Keyword, i, i+3)
		return l.nested(i+3, shMath, chroma.Keyword, 2)
	case strings.HasPrefix(s[i:], "$("):
		l.emit(chroma.Keyword, i, i+2)
		return l.nested(i+2, shParen, chroma.Keyword, 1)
	case strings.HasPrefix(s[i:], "${"):
		e := i + 2
		if e < len(s) && s[e] == '#' {
			e++
		}
		l.emit(chroma.LiteralStringInterpol, i, e)
		return l.nested(e, shCurly, chroma.LiteralStringInterpol, 1)
	}
	if i+1 < len(s) {
		c := s[i+1]
		switch {
		case c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z':
			e := wordEnd(s, i+1, isWord)
			l.emit(chroma.NameVariable, i, e)
			return e
		case isDigit(c):
			e := i + 1
			for e < len(s) && isDigit(s[e]) {
				e++
			}
			l.emit(chroma.NameVariable, i, e)
			return e
		case strings.IndexByte("#$?!_*@-", c) >= 0:
			l.emit(chroma.NameVariable, i, i+2)
			return i + 2
		}
	}
	l.emit(chroma.Text, i, i+1)
	return i + 1
}

// nested lexes a state from i to its closer, which it emits as t with
// width n.
func (l *shLex) nested(i, st int, t chroma.TokenType, n int) int {
	s := l.src
	for i < len(s) {
		k := l.step(i, st)
		if k < 0 {
			k = -1 - k
			l.emit(t, k, k+n)
			return k + n
		}
		i = k
	}
	return i
}