bench_test.go

@@ -105,7 +105,7 @@ func BenchmarkCodePage(b *testing.B) {  // BenchmarkSyntaxTree times a tree page's first view on jquery.js in its // stages: the helper with reading its output, rendering the rows, and-// the whole page as a request returns it. It skips where the helper or+// the whole page as a request returns it, where the two overlap. It skips where the helper or // the file is missing. func BenchmarkSyntaxTree(b *testing.B) { 	src, err := os.ReadFile("/usr/share/javascript/jquery/jquery.js")@@ -115,19 +115,23 @@ func BenchmarkSyntaxTree(b *testing.B) { 	if _, err := syntaxHelper(); err != nil { 		b.Skip("no helper; make syntax builds it") 	}-	nodes, err := parseSyntax(context.Background(), "jquery.js", src)+	nodes, err := syntaxNodes(context.Background(), "jquery.js", src) 	if err != nil { 		b.Fatal(err) 	} 	b.Run("helper", func(b *testing.B) { 		for b.Loop() {-			parseSyntax(context.Background(), "jquery.js", src)+			parseSyntax(context.Background(), "jquery.js", src, func(synNode) {}) 		} 	}) 	b.Run("rows", func(b *testing.B) { 		b.ReportAllocs() 		for b.Loop() {-			renderSyntax(nodes, "jquery.js", src)+			w := newSynRows("jquery.js", src)+			for _, n := range nodes {+				w.add(n)+			}+			w.done() 		} 	}) 	b.Run("page", func(b *testing.B) {

semantic.go

@@ -97,7 +97,7 @@ func scopeKind(kind string) bool { // helper's syntax tree. Locals, such as a function's variables, are left // out; a member is qualified by its container, eg Parser.parse. func declarations(ctx context.Context, name string, src []byte) ([]decl, error) {-	nodes, err := parseSyntax(ctx, name, src)+	nodes, err := syntaxNodes(ctx, name, src) 	if err != nil { 		return nil, err 	}

server.go

@@ -1098,7 +1098,12 @@ func (t *tree) serveCode(w http.ResponseWriter, r *http.Request, name string, in 	if _, ok := q["tree"]; ok { 		tree, err := syntaxPage(r.Context(), name, pageHref(r.URL.Path), b) 		if err == nil {-			body.WriteString(`<div class="code">` + codeViews(name, "tree") + tree + `</div>`)+			// A large file's tree is megabytes, so it is written once,+			// into room made for it.+			body.Grow(len(tree) + 1024)+			body.WriteString(`<div class="code">` + codeViews(name, "tree"))+			body.WriteString(tree)+			body.WriteString(`</div>`) 			t.render(w, r, http.StatusOK, page{Title: name, Crumbs: t.crumbs(name), Kind: "tree", Body: template.HTML(body.String())}) 			return 		}@@ -1322,6 +1327,8 @@ func render(w http.ResponseWriter, r *http.Request, status int, p page) { 			Commit: buildCommit, Rendered: rendered(r)} 	} 	var b bytes.Buffer+	// The body is most of the page, and can be megabytes.+	b.Grow(len(p.Body) + 16<<10) 	if err := layout.ExecuteTemplate(&b, "page", p); err != nil { 		http.Error(w, err.Error(), http.StatusInternalServerError) 		return

syntax.go

@@ -1,11 +1,13 @@ package main  import (+	"bufio" 	"bytes" 	"context" 	"crypto/sha256" 	"errors" 	"html/template"+	"io" 	"net/url" 	"os" 	"os/exec"@@ -59,57 +61,114 @@ func syntaxHelper() (string, error) { }  // parseSyntax runs the helper on src, which name's extension picks the-// grammar for.-func parseSyntax(ctx context.Context, name string, src []byte) ([]synNode, error) {+// grammar for, and hands row each node as the helper prints it. The+// helper prints while it walks the tree, so rows render in the meantime.+func parseSyntax(ctx context.Context, name string, src []byte, row func(synNode)) error { 	helper, err := syntaxHelper() 	if err != nil {-		return nil, err+		return err 	} 	ctx, cancel := context.WithTimeout(ctx, syntaxWait) 	defer cancel() 	cmd := exec.CommandContext(ctx, helper, filepath.Base(name)) 	cmd.Stdin = bytes.NewReader(src)-	var out, stderr bytes.Buffer-	cmd.Stdout, cmd.Stderr = &out, &stderr-	if err := cmd.Run(); err != nil {+	var stderr bytes.Buffer+	cmd.Stderr = &stderr+	out, err := cmd.StdoutPipe()+	if err != nil {+		return err+	}+	if err := cmd.Start(); err != nil {+		return err+	}+	if err := readNodes(out, row); err != nil {+		// Cancelling kills the helper, so Wait returns.+		cancel()+		cmd.Wait()+		return err+	}+	if err := cmd.Wait(); err != nil { 		var ee *exec.ExitError 		if errors.As(err, &ee) && syscall.Errno(ee.ExitCode()) == syscall.EOPNOTSUPP {-			return nil, errNoGrammar+			return errNoGrammar 		} 		if msg := strings.TrimSpace(stderr.String()); msg != "" {-			return nil, errors.New(msg)+			return errors.New(msg) 		}-		return nil, err+		return err 	}-	return readNodes(out.String())+	return nil } -// readNodes reads the helper's output: a node a line, with its depth,-// kind, field and range as tab-separated fields. A large file prints tens-// of thousands of lines, so it cuts fields in place, with no slice per-// line.-func readNodes(out string) ([]synNode, error) {-	nodes := make([]synNode, 0, strings.Count(out, "\n"))-	for line := range strings.Lines(out) {-		line = strings.TrimSuffix(line, "\n")-		if k := strings.Count(line, "\t"); k != 6 {-			return nil, errors.New("the helper printed a line of " + strconv.Itoa(k+1) + " fields")+// syntaxNodes is every node parseSyntax hands on, in order.+func syntaxNodes(ctx context.Context, name string, src []byte) ([]synNode, error) {+	var nodes []synNode+	err := parseSyntax(ctx, name, src, func(n synNode) { nodes = append(nodes, n) })+	return nodes, err+}++// readNodes reads the helper's output from r: a node a line, with its+// depth, kind, field and range as tab-separated fields. A large file+// prints tens of thousands of lines, so it cuts fields in place and keeps+// one copy of each kind and field name, which the grammar holds to a few+// hundred.+func readNodes(r io.Reader, row func(synNode)) error {+	br := bufio.NewReaderSize(r, 64<<10)+	names := map[string]string{}+	intern := func(b []byte) string {+		if s, ok := names[string(b)]; ok {+			return s 		}-		var f [7]string+		s := string(b)+		names[s] = s+		return s+	}+	for {+		line, err := br.ReadSlice('\n')+		if errors.Is(err, bufio.ErrBufferFull) {+			return errors.New("the helper printed a line over 64 KiB")+		}+		if err != nil && err != io.EOF {+			return err+		}+		if len(line) == 0 {+			return nil+		}+		line = bytes.TrimSuffix(line, []byte("\n"))+		if k := bytes.Count(line, []byte("\t")); k != 6 {+			return errors.New("the helper printed a line of " + strconv.Itoa(k+1) + " fields")+		}+		var f [7][]byte 		for i := range 6 {-			f[i], line, _ = strings.Cut(line, "\t")+			f[i], line, _ = bytes.Cut(line, []byte("\t")) 		} 		f[6] = line 		var v [5]int-		for i, s := range [5]string{f[0], f[3], f[4], f[5], f[6]} {-			var err error-			if v[i], err = strconv.Atoi(s); err != nil {-				return nil, errors.New("the helper printed a bad number: " + s)+		for i, b := range [5][]byte{f[0], f[3], f[4], f[5], f[6]} {+			var ok bool+			if v[i], ok = atoi(b); !ok {+				return errors.New("the helper printed a bad number: " + string(b)) 			} 		}-		nodes = append(nodes, synNode{depth: v[0], kind: f[1], field: f[2], sr: v[1], sc: v[2], er: v[3], ec: v[4]})+		row(synNode{depth: v[0], kind: intern(f[1]), field: intern(f[2]), sr: v[1], sc: v[2], er: v[3], ec: v[4]}) 	}-	return nodes, nil+}++// atoi reads b as a number of up to 18 digits, the most an int64 always+// holds, with no sign. strconv.Atoi would need a string, which costs an+// allocation a field.+func atoi(b []byte) (int, bool) {+	if len(b) == 0 || len(b) > 18 {+		return 0, false+	}+	n := 0+	for _, c := range b {+		if c < '0' || c > '9' {+			return 0, false+		}+		n = n*10 + int(c-'0')+	}+	return n, true }  // syntaxCache keeps rendered trees by file name and content hash, as@@ -127,86 +186,115 @@ func syntaxPage(ctx context.Context, name, href string, src []byte) (string, err 	if h, ok := syntaxCache.get(key); ok { 		return h.(string), nil 	}-	nodes, err := parseSyntax(ctx, name, src)-	if err != nil {+	rows := newSynRows(href, src)+	if err := parseSyntax(ctx, name, src, rows.add); err != nil { 		return "", err 	}-	h := renderSyntax(nodes, href, src)-	syntaxCache.put(key, h, int64(len(h)))+	h := rows.done()+	// The rows keep the builder's whole buffer, which the guess in+	// newSynRows may make larger than h.+	syntaxCache.put(key, h, int64(rows.b.Cap())) 	return h, nil } -// renderSyntax writes the rows of nodes, parsed from src, with line links-// to href.-func renderSyntax(nodes []synNode, href string, src []byte) string {-	lines := bytes.Split(src, []byte("\n"))-	var b strings.Builder-	// About 120 bytes a row, so the builder grows about once.-	b.Grow(len(nodes) * 128)-	b.WriteString("<div class=\"syntax\">\n")-	var num, num2 [20]byte-	open := 0 // folds left open-	for i, n := range nodes {-		for ; open > n.depth; open-- {-			b.WriteString("</details>\n")-		}-		folds := i+1 < len(nodes) && nodes[i+1].depth > n.depth-		switch {-		case folds && n.depth == 0:-			b.WriteString("<details open><summary>")-		case folds:-			b.WriteString("<details><summary>")-		default:-			b.WriteString("<div>")-		}-		// The row: its field in its parent, its kind, its text, and its-		// lines as a link. Each row repeats these tags, so they are short.-		if n.field != "" {-			b.WriteString("<i>")-			escapeHTML(&b, n.field)-			b.WriteString(":</i> ")-		}-		b.WriteString("<b>")-		escapeHTML(&b, n.kind)-		b.WriteString("</b>")-		writeExcerpt(&b, lines, n)-		er := n.er-		// A node that ends at the start of a row ends on the row before.-		if n.ec == 0 && er > n.sr {-			er---		}-		first := strconv.AppendInt(num[:0], int64(n.sr+1), 10)-		var last []byte-		if er > n.sr {-			last = strconv.AppendInt(num2[:0], int64(er+1), 10)-		}-		b.WriteString(` <a href="`)-		escapeHTML(&b, href)-		b.WriteString("#L")-		b.Write(first)-		if last != nil {-			b.WriteString("-L")-			b.Write(last)-		}-		b.WriteString(`">`)-		b.Write(first)-		if last != nil {-			b.WriteString("–")-			b.Write(last)-		}-		b.WriteString("</a>")-		if folds {-			b.WriteString("</summary>\n")-			open++-		} else {-			b.WriteString("</div>\n")-		}+// synRows writes a syntax tree's rows as its nodes come. It holds each+// node back until the next one shows whether it folds.+type synRows struct {+	b     strings.Builder+	lines [][]byte+	href  string+	open  int // folds left open+	held  synNode+	any   bool // whether a node is held+}++// newSynRows starts the rows of src's tree, with line links to href.+func newSynRows(href string, src []byte) *synRows {+	w := &synRows{lines: bytes.Split(src, []byte("\n")), href: href}+	// A row is about 120 bytes, and a line of a large file gives about+	// four, so the builder seldom grows where it would cost the most.+	w.b.Grow(len(w.lines) * 512)+	w.b.WriteString("<div class=\"syntax\">\n")+	return w+}++func (w *synRows) add(n synNode) {+	if w.any {+		w.row(w.held, n.depth > w.held.depth)+	}+	w.held, w.any = n, true+}++// done writes the last row, closes the folds and returns the rows.+func (w *synRows) done() string {+	if w.any {+		w.row(w.held, false)+	}+	for ; w.open > 0; w.open-- {+		w.b.WriteString("</details>\n") 	}-	for ; open > 0; open-- {+	w.b.WriteString("</div>\n")+	return w.b.String()+}++// row writes n, as a fold's summary when folds says the next node is its+// child.+func (w *synRows) row(n synNode, folds bool) {+	b := &w.b+	for ; w.open > n.depth; w.open-- { 		b.WriteString("</details>\n") 	}-	b.WriteString("</div>\n")-	return b.String()+	switch {+	case folds && n.depth == 0:+		b.WriteString("<details open><summary>")+	case folds:+		b.WriteString("<details><summary>")+	default:+		b.WriteString("<div>")+	}+	// The row: its field in its parent, its kind, its text, and its lines+	// as a link. Each row repeats these tags, so they are short.+	if n.field != "" {+		b.WriteString("<i>")+		escapeHTML(b, n.field)+		b.WriteString(":</i> ")+	}+	b.WriteString("<b>")+	escapeHTML(b, n.kind)+	b.WriteString("</b>")+	writeExcerpt(b, w.lines, n)+	er := n.er+	// A node that ends at the start of a row ends on the row before.+	if n.ec == 0 && er > n.sr {+		er--+	}+	var num, num2 [20]byte+	first := strconv.AppendInt(num[:0], int64(n.sr+1), 10)+	var last []byte+	if er > n.sr {+		last = strconv.AppendInt(num2[:0], int64(er+1), 10)+	}+	b.WriteString(` <a href="`)+	escapeHTML(b, w.href)+	b.WriteString("#L")+	b.Write(first)+	if last != nil {+		b.WriteString("-L")+		b.Write(last)+	}+	b.WriteString(`">`)+	b.Write(first)+	if last != nil {+		b.WriteString("–")+		b.Write(last)+	}+	b.WriteString("</a>")+	if folds {+		b.WriteString("</summary>\n")+		w.open+++	} else {+		b.WriteString("</div>\n")+	} }  // excerptLen is how many characters of a node's text a row shows.

syntax_test.go

@@ -47,15 +47,35 @@ func TestSyntaxPage(t *testing.T) { // Without the helper, or without a grammar, a tree page says so. func TestSyntaxMissing(t *testing.T) { 	fakeSyntax(t)-	if _, err := parseSyntax(context.Background(), "x.txt", nil); !errors.Is(err, errNoGrammar) {+	if _, err := syntaxNodes(context.Background(), "x.txt", nil); !errors.Is(err, errNoGrammar) { 		t.Errorf("no grammar: %v", err) 	} 	t.Setenv("PATH", t.TempDir())-	if _, err := parseSyntax(context.Background(), "b.go", nil); !errors.Is(err, errNoHelper) {+	if _, err := syntaxNodes(context.Background(), "b.go", nil); !errors.Is(err, errNoHelper) { 		t.Errorf("no helper: %v", err) 	} } +// Output the helper should never print fails the page with what was+// wrong, and so does a helper that dies after its first rows.+func TestSyntaxBadOutput(t *testing.T) {+	for _, tc := range []struct{ script, want string }{+		{`printf '0\tsource_file\t\t0\t0\n'`, "a line of 5 fields"},+		{`printf '0\tsource_file\t\t0\t-1\t3\t0\n'`, "a bad number: -1"},+		{`printf '0\tsource_file\t\t0\t0\t3\t0\n'; echo 'E5: broke' >&2; exit 5`, "E5: broke"},+	} {+		dir := t.TempDir()+		script := "#!/bin/sh\ncat >/dev/null\n" + tc.script + "\n"+		if err := os.WriteFile(filepath.Join(dir, progName+"-syntax"), []byte(script), 0o755); err != nil {+			t.Fatal(err)+		}+		t.Setenv("PATH", dir)+		if _, err := syntaxPage(context.Background(), "a.go", "a.go", []byte("x\n")); err == nil || !strings.Contains(err.Error(), tc.want) {+			t.Errorf("%s: %v, want %q", tc.script, err, tc.want)+		}+	}+}+ // An excerpt keeps a node's first row, its spaces folded, and marks text // cut or continued. func TestExcerpt(t *testing.T) {