package main
import (
"bytes"
"crypto/sha256"
"html/template"
"io"
"net/url"
"path"
"strconv"
"strings"
"sync"
"github.com/alecthomas/chroma/v2"
"github.com/alecthomas/chroma/v2/lexers"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
east "github.com/yuin/goldmark/extension/ast"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
)
// markdown renders docs as CommonMark with GitHub's extensions: lines join
// into paragraphs, bare URLs become links and a list may follow a line
// with no blank line between. Wiki links, [[name]], become links too. Raw HTML passes through; the
// Content-Security-Policy keeps its scripts from running.
var markdown = goldmark.New(
goldmark.WithExtensions(extension.GFM, extension.Footnote, extension.DefinitionList, extension.Typographer),
goldmark.WithParserOptions(parser.WithAutoHeadingID(), parser.WithInlineParsers(util.Prioritized(wikiLinkParser{}, 199))),
goldmark.WithRendererOptions(
html.WithUnsafe(),
// The lowest priority registers last and wins, so this replaces
// goldmark's fenced code renderer at 1000 and its task list
// renderer at 500.
renderer.WithNodeRenderers(util.Prioritized(nodeRenderer{}, 100)),
),
)
// renderMarkdown renders a doc. missing, when given, reports whether a
// relative link's path, unescaped and without query or fragment, names
// nothing; such a link gets the class broken and a title that says so.
// web, when given, is where the web address serves, eg
// http://127.31.1.2:8080 or "" on the web address itself; a link to a
// repo on a forge, or a file in one, is followed by a mark that reads it
// there.
func renderMarkdown(src []byte, missing func(dest string) bool, web *string) (template.HTML, error) {
doc := markdown.Parser().Parse(text.NewReader(src))
markAlerts(doc, src)
if missing != nil {
markBroken(doc, missing)
}
if web != nil {
markWeb(doc, src, *web)
}
var out bytes.Buffer
if err := markdown.Renderer().Render(&out, src, doc); err != nil {
return "", err
}
return template.HTML(out.String()), nil
}
// markBroken marks each link whose relative target is missing.
func markBroken(doc ast.Node, missing func(string) bool) {
ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
l, ok := n.(*ast.Link)
if !entering || !ok {
return ast.WalkContinue, nil
}
dest := relativePath(string(l.Destination))
if dest == "" || !missing(dest) {
return ast.WalkContinue, nil
}
l.SetAttributeString("class", []byte("broken"))
if len(l.Title) == 0 {
l.SetAttributeString("title", []byte("missing: "+dest))
}
return ast.WalkContinue, nil
})
}
// alertKinds are the words a quote may open with as [!WORD] to become an
// alert.
var alertKinds = map[string]bool{"note": true, "tip": true, "important": true, "warning": true, "caution": true}
// alertLabel names an alert's kind at the top of its quote.
type alertLabel struct {
ast.BaseBlock
kind string
}
var kindAlertLabel = ast.NewNodeKind("AlertLabel")
func (n *alertLabel) Kind() ast.NodeKind { return kindAlertLabel }
func (n *alertLabel) IsRaw() bool { return true }
func (n *alertLabel) Dump(src []byte, level int) { ast.DumpHelper(n, src, level, nil, nil) }
// markAlerts turns each quote whose first line is only [!NOTE], [!TIP],
// [!IMPORTANT], [!WARNING] or [!CAUTION], in any case, into an alert: the
// marker gives way to a label that names the kind.
func markAlerts(doc ast.Node, src []byte) {
ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
q, ok := n.(*ast.Blockquote)
if !entering || !ok {
return ast.WalkContinue, nil
}
p, ok := q.FirstChild().(*ast.Paragraph)
if !ok || p.Lines().Len() == 0 {
return ast.WalkContinue, nil
}
first := p.Lines().At(0)
marker := strings.TrimSpace(string(first.Value(src)))
word := strings.ToLower(strings.TrimSuffix(strings.TrimPrefix(marker, "[!"), "]"))
if len(marker) != len(word)+3 || !alertKinds[word] {
return ast.WalkContinue, nil
}
// The marker is text on the first line, and nothing else is.
for c := p.FirstChild(); c != nil; {
t, ok := c.(*ast.Text)
if !ok || t.Segment.Start >= first.Stop {
break
}
next := c.NextSibling()
p.RemoveChild(p, c)
c = next
}
if p.ChildCount() == 0 {
q.RemoveChild(q, p)
}
q.SetAttributeString("class", []byte("alert "+word))
q.InsertBefore(q, q.FirstChild(), &alertLabel{kind: word})
return ast.WalkSkipChildren, nil
})
}
// renderAlertLabel writes the label as text of www's own.
func renderAlertLabel(w util.BufWriter, src []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
if entering {
w.WriteString(`<p class="own alert">` + node.(*alertLabel).kind + "</p>\n")
}
return ast.WalkContinue, nil
}
// webMark follows a link to a forge with a link that reads it on the web
// address, or, on the web address, a link that reads it there with one
// back to the forge.
type webMark struct {
ast.BaseInline
href, key string
// forge, when set, names the forge the mark leads to, eg github, and
// key is the host it opens, eg raw.githubusercontent.com.
forge string
}
var kindWebMark = ast.NewNodeKind("WebMark")
func (n *webMark) Kind() ast.NodeKind { return kindWebMark }
func (n *webMark) Dump(src []byte, level int) { ast.DumpHelper(n, src, level, nil, nil) }
// markWeb puts a webMark after each link, bare URLs included, that names a
// repo on a forge or a file in one. On the web address itself, base is ""
// and the link turns around: it reads the repo there, so a reader in an
// online repo stays in www, and the mark leads to the forge.
func markWeb(doc ast.Node, src []byte, base string) {
type found struct {
link ast.Node
u *url.URL
key string
href string
}
var links []found
ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
var dest string
switch l := n.(type) {
case *ast.Link:
dest = string(l.Destination)
case *ast.AutoLink:
dest = string(l.URL(src))
default:
return ast.WalkContinue, nil
}
if !entering {
return ast.WalkContinue, nil
}
u, err := url.Parse(dest)
if err != nil {
return ast.WalkSkipChildren, nil
}
if key, p := webPath(u); p != "" {
links = append(links, found{n, u, key, base + (&url.URL{Path: p, Fragment: u.Fragment}).String()})
}
return ast.WalkSkipChildren, nil
})
// The tree changes after the walk, which would stop at a node that
// left it.
for _, f := range links {
n, parent := f.link, f.link.Parent()
if base != "" {
parent.InsertAfter(parent, n, &webMark{href: f.href, key: f.key})
continue
}
switch l := n.(type) {
case *ast.Link:
l.Destination = []byte(f.href)
case *ast.AutoLink:
link := ast.NewLink()
link.Destination = []byte(f.href)
link.AppendChild(link, ast.NewString(l.Label(src)))
parent.ReplaceChild(parent, n, link)
n = link
}
forge, _, _ := strings.Cut(f.key, ".")
parent.InsertAfter(parent, n, &webMark{href: f.u.String(), key: strings.ToLower(f.u.Hostname()), forge: forge})
}
}
// renderWebMark writes the mark after the link: the program's name, or on
// the web address the forge's, eg github.
func renderWebMark(w util.BufWriter, src []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
if entering {
m := node.(*webMark)
esc := template.HTMLEscapeString
if m.forge != "" {
w.WriteString(`<a class="web" href="` + esc(m.href) + `" title="open on ` + esc(m.key) + `">` + esc(m.forge) + `</a>`)
return ast.WalkContinue, nil
}
w.WriteString(`<a class="web" href="` + esc(m.href) + `" title="read ` + esc(m.key) + ` in ` + progName + `">` + progName + `</a>`)
}
return ast.WalkContinue, nil
}
// relativePath is the path a relative link names, unescaped, or "" for a
// link with a scheme or host, an absolute path or only a fragment or
// query.
func relativePath(dest string) string {
u, err := url.Parse(dest)
if err != nil || u.Scheme != "" || u.Host != "" || u.Opaque != "" || strings.HasPrefix(u.Path, "/") {
return ""
}
return u.Path
}
// nodeRenderer replaces goldmark's rendering of fenced code and task list
// checkboxes, and renders the nodes www adds.
type nodeRenderer struct{}
func (nodeRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) {
r.Register(ast.KindFencedCodeBlock, renderFence)
r.Register(east.KindTaskCheckBox, renderCheckBox)
r.Register(kindWebMark, renderWebMark)
r.Register(kindAlertLabel, renderAlertLabel)
}
// renderCheckBox leaves out goldmark's disabled attribute, which greys the
// box out, so a task list looks as it did under pandoc.
func renderCheckBox(w util.BufWriter, src []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
if node.(*east.TaskCheckBox).IsChecked {
w.WriteString(`<input type="checkbox" checked=""> `)
} else {
w.WriteString(`<input type="checkbox"> `)
}
return ast.WalkContinue, nil
}
// renderFence draws a Graphviz block through dot and highlights a fenced
// block whose language chroma knows. A graph dot cannot draw shows as code
// under a line that says why; a block in any other language is plain
// <pre><code>.
func renderFence(w util.BufWriter, src []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
n := node.(*ast.FencedCodeBlock)
var code strings.Builder
lines := n.Lines()
for i := 0; i < lines.Len(); i++ {
seg := lines.At(i)
code.Write(seg.Value(src))
}
var lexer chroma.Lexer
if lang := n.Language(src); lang != nil {
if isGraph(string(lang)) {
svg, err := drawGraph(code.String())
if err == nil {
w.WriteString(string(svg) + "\n")
return ast.WalkSkipChildren, nil
}
w.WriteString(`<p class="own warn">` + template.HTMLEscapeString(err.Error()) + "</p>\n")
}
lexer = lexers.Get(string(lang))
}
if lexer == nil {
w.WriteString("<pre><code>")
w.WriteString(template.HTMLEscapeString(code.String()))
w.WriteString("</code></pre>\n")
return ast.WalkSkipChildren, nil
}
highlight(w, lexer, code.String(), false, false)
w.WriteString("\n")
return ast.WalkSkipChildren, nil
}
// codeLexer picks the lexer for a code page by file name, which covers
// Makefile and Dockerfile as well as extensions, and falls back to plain
// text. lexers.Match tries every glob of every lexer, about 3 ms, so the
// answer is kept per base name.
func codeLexer(name string) chroma.Lexer {
base := path.Base(name)
if l, ok := lexerByName.Load(base); ok {
return l.(chroma.Lexer)
}
l := lexers.Match(base)
if l == nil {
l = lexers.Fallback
}
lexerByName.Store(base, l)
return l
}
var lexerByName sync.Map
// tokenise lexes Go with the standard library's scanner, Python,
// JavaScript and shell with the program's own lexers, and everything else
// with chroma.
func tokenise(lexer chroma.Lexer, code string) []chroma.Token {
if lexer == lexers.Go {
return goTokens(code)
}
if own := ownLexers[lexer]; own != nil {
return own(ensureLF(code))
}
it, err := chroma.Coalesce(lexer).Tokenise(nil, code)
if err != nil {
return []chroma.Token{{Type: chroma.Text, Value: code}}
}
return it.Tokens()
}
// warmLexers compiles the lexers for common files in the background, so
// the first code page a server shows does not wait for chroma to build its
// regular expressions.
func warmLexers() {
for _, name := range []string{"x.sh", "Makefile", "x.py", "x.js", "x.ts", "x.json", "x.yaml", "x.toml",
"x.html", "x.css", "x.c", "x.rs", "Dockerfile", "x.md"} {
tokenise(codeLexer(name), "x\n")
}
}
// highlight writes code in pandoc's markup with pandoc's short classes, so
// the stylesheet's colours apply unchanged. Chroma only lexes. Numbered
// lines carry the ids GitHub uses, so main.go#L42 lands on line 42.
// Escaping keeps file content from forging an id. Scoped lines, which
// must be numbered, draw each block of code as a box behind it.
func highlight(w io.Writer, lexer chroma.Lexer, code string, numbered, scoped bool) {
// Every write goes to one builder: a builder of its own, or w when w is
// one, as for a code page, so the page is never copied whole.
b, ok := w.(*strings.Builder)
if !ok {
b = new(strings.Builder)
defer func() { io.WriteString(w, b.String()) }()
}
lang := langName(lexer)
lines := splitLines(tokenise(lexer, code))
// A final newline ends the last line; it does not start another.
if len(lines) > 0 && blank(lines[len(lines)-1]) {
lines = lines[:len(lines)-1]
}
var scopes []scope
if numbered && scoped {
scopes = findScopes(lines, lang == "python")
}
pre := "sourceCode " + lang
if numbered {
pre = "sourceCode numberSource " + lang + " numberLines"
}
if len(scopes) > 0 {
pre += " scopes"
}
b.Grow(pageSize(code, lines, len(scopes), numbered))
b.WriteString(`<div class="sourceCode"><pre class="` + strings.Join(strings.Fields(pre), " ") + `"><code class="` + strings.TrimSpace("sourceCode "+lang) + `">`)
var num [20]byte
for i, line := range lines {
if i > 0 {
b.WriteByte('\n')
}
if numbered {
id := strconv.AppendInt(num[:0], int64(i+1), 10)
b.WriteString(`<span id="L`)
b.Write(id)
b.WriteString(`"><a href="#L`)
b.Write(id)
b.WriteString(`"></a>`)
n := 0
for n < len(scopes) && scopes[n].first == i {
n++
}
writeScopes(b, scopes[:n])
scopes = scopes[n:]
} else {
b.WriteString("<span>")
}
writeLine(b, line)
b.WriteString("</span>")
}
b.WriteString("</code></pre></div>")
}
// pageSize guesses the length of highlight's output, so the builder grows
// once. Each token is weighed at 16 bytes, as if about half carried a
// span of their own; on this repo's code and Debian's jquery.js,
// bash_completion, argparse.py and stdio.h the guess is 7 to 41% over.
// Slack stays in a cached page unseen, and a builder that doubles leaves
// up to 100%.
func pageSize(code string, lines [][]chroma.Token, scopes int, numbered bool) int {
tokens := 0
for _, line := range lines {
tokens += len(line)
}
perLine := len("\n<span></span>")
if numbered {
perLine = len(`\n<span id="L1234"><a href="#L1234"></a></span>`)
}
box := len(`<i class="scope leaf shade" style="--c:12;--w:34;--d:3"></i>`)
return len(code) + len(code)/8 + tokens*16 + len(lines)*perLine + scopes*box
}
// splitLines splits tokens into lines as chroma.SplitTokensIntoLines
// does: a token that spans lines becomes one per line, each ending in its
// newline. Chroma allocates each line on its own, which took a third of
// the page writer; here one array backs every line.
func splitLines(tokens []chroma.Token) [][]chroma.Token {
n, rows := len(tokens), 1
for _, tok := range tokens {
c := strings.Count(tok.Value, "\n")
n += c
rows += c
}
all := make([]chroma.Token, 0, n)
lines := make([][]chroma.Token, 0, rows)
start := 0
tokens:
for _, tok := range tokens {
for {
i := strings.IndexByte(tok.Value, '\n')
if i < 0 {
break
}
head := tok
head.Value = tok.Value[:i+1]
all = append(all, head)
lines = append(lines, all[start:len(all):len(all)])
start = len(all)
// An empty tail starts nothing.
if tok.Value = tok.Value[i+1:]; tok.Value == "" {
continue tokens
}
}
all = append(all, tok)
}
if start < len(all) {
lines = append(lines, all[start:len(all):len(all)])
}
// Chroma drops a last line that holds only an empty token.
if len(lines) > 0 {
if last := lines[len(lines)-1]; len(last) == 1 && last[0].Value == "" {
lines = lines[:len(lines)-1]
}
}
return lines
}
// codeCache keeps highlighted code pages by lexer and content hash. Chroma
// lexes most languages at about 70 µs a line, so a large Python or
// JavaScript file takes far over the page budget; from the cache a page
// takes as long as a Go one.
var codeCache = newCache(16 << 20)
// highlightPage is highlight for a code page, through codeCache.
func highlightPage(lexer chroma.Lexer, b []byte, scoped bool) string {
sum := sha256.Sum256(b)
key := langName(lexer) + "\x00" + strconv.FormatBool(scoped) + "\x00" + string(sum[:])
if h, ok := codeCache.get(key); ok {
return h.(string)
}
var out strings.Builder
highlight(&out, lexer, string(b), true, scoped)
h := out.String()
codeCache.put(key, h, int64(len(h)))
return h
}
// writeLine writes one line's tokens, merging neighbours of one class.
func writeLine(b *strings.Builder, line []chroma.Token) {
class := ""
for _, tok := range line {
v := strings.TrimSuffix(tok.Value, "\n")
if v == "" {
continue
}
if c := pandocClass(tok.Type); c != class {
if class != "" {
b.WriteString("</span>")
}
if c != "" {
b.WriteString(`<span class="` + c + `">`)
}
class = c
}
escapeHTML(b, v)
}
if class != "" {
b.WriteString("</span>")
}
}
// escapeHTML writes s escaped as template.HTMLEscapeString escapes it,
// without making a string of its own.
func escapeHTML(b *strings.Builder, s string) {
last := 0
for i := 0; i < len(s); i++ {
var esc string
switch s[i] {
case '"':
esc = """
case '\'':
esc = "'"
case '&':
esc = "&"
case '<':
esc = "<"
case '>':
esc = ">"
case 0:
esc = "\uFFFD"
default:
continue
}
b.WriteString(s[last:i])
b.WriteString(esc)
last = i + 1
}
b.WriteString(s[last:])
}
// blank reports whether a line holds no text.
func blank(line []chroma.Token) bool {
for _, tok := range line {
if tok.Value != "" {
return false
}
}
return true
}
// langName is the lexer's class on the code element: its first alias, eg
// go or makefile. Plain text has none.
func langName(l chroma.Lexer) string {
c := l.Config()
if l == lexers.Fallback || c == nil {
return ""
}
name := c.Name
if len(c.Aliases) > 0 {
name = c.Aliases[0]
}
name = strings.ToLower(name)
if strings.ContainsFunc(name, func(r rune) bool {
return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '+' || r == '_' || r == '-')
}) {
return ""
}
return name
}
// pandocClass maps a chroma token type to the class pandoc gives the same
// kind of token. Types without a class render as plain text.
func pandocClass(t chroma.TokenType) string {
switch t {
case chroma.KeywordType, chroma.NameClass:
return "dt"
case chroma.KeywordConstant, chroma.NameConstant:
return "cn"
case chroma.NameTag:
return "kw"
case chroma.NameAttribute:
return "at"
case chroma.LiteralStringChar:
return "ch"
case chroma.LiteralStringEscape:
return "sc"
case chroma.LiteralStringDoc:
return "do"
case chroma.LiteralNumberFloat:
return "fl"
case chroma.LiteralNumberHex, chroma.LiteralNumberOct, chroma.LiteralNumberBin:
return "bn"
case chroma.Error:
return "er"
}
switch t.SubCategory() {
case chroma.NameBuiltin:
return "bu"
case chroma.NameFunction:
return "fu"
case chroma.NameVariable:
return "va"
case chroma.LiteralString:
return "st"
case chroma.LiteralNumber:
return "dv"
case chroma.CommentPreproc:
return "pp"
}
switch t.Category() {
case chroma.Keyword:
return "kw"
case chroma.Operator, chroma.Punctuation:
return "op"
case chroma.Comment:
return "co"
}
return ""
}