syntax/main.go

@@ -11,28 +11,33 @@ // main binary runs it for each tree page and does without it when it is // absent. The binary takes its name from the Makefile, eg www-syntax. //+// The parse and the walk run in C, in walk.c, in one call; the Go binding+// supplies tree-sitter's library and the grammars.+// // Usage: <name>-syntax FILE < source package main  /* #include <stddef.h>+#include <stdint.h>  void ts_set_allocator(void *(*)(size_t), void *(*)(size_t, size_t), void *(*)(void *, size_t), void (*)(void *));+int syntax_tree(const void *lang, const char *src, uint32_t len); */ import "C"  import (-	"bufio" 	"fmt" 	"io"+	"math" 	"os" 	"path/filepath"-	"strconv" 	"strings" 	"syscall" 	"unsafe" -	sitter "github.com/tree-sitter/go-tree-sitter"+	// The binding builds tree-sitter's C library, which walk.c calls.+	_ "github.com/tree-sitter/go-tree-sitter" 	bash "github.com/tree-sitter/tree-sitter-bash/bindings/go" 	csharp "github.com/tree-sitter/tree-sitter-c-sharp/bindings/go" 	c "github.com/tree-sitter/tree-sitter-c/bindings/go"@@ -104,84 +109,21 @@ func main() { 	if err != nil { 		os.Exit(fail(syscall.EIO, "cannot read the source: "+err.Error())) 	}-	p := sitter.NewParser()-	defer p.Close()-	if err := p.SetLanguage(sitter.NewLanguage(lang())); err != nil {-		os.Exit(fail(syscall.EINVAL, "cannot load the grammar: "+err.Error()))-	}-	tree := p.Parse(src, nil)-	if tree == nil {-		os.Exit(fail(syscall.EIO, "the parser gave no tree"))-	}-	defer tree.Close()-	w := bufio.NewWriter(os.Stdout)-	write(w, tree)-	if err := w.Flush(); err != nil {-		os.Exit(fail(syscall.EIO, "cannot write the tree: "+err.Error()))+	if len(src) > math.MaxUint32 {+		os.Exit(fail(syscall.EFBIG, "the source is over 4 GiB")) 	}-}--// write prints the named nodes in preorder. Anonymous nodes, eg-// punctuation and keywords, are left out, but their named descendants-// keep their place under the nearest named ancestor. Each call into-// tree-sitter crosses cgo, so a node costs as few as the line needs, and-// kinds and field names are looked up once per id.-func write(w *bufio.Writer, tree *sitter.Tree) {-	lang := tree.Language()-	kinds := map[uint16]string{}-	fields := map[uint16]string{}-	cur := tree.Walk()-	defer cur.Close()-	depth := 0-	named := []int{} // the depths of the named nodes above the cursor-	for {-		n := cur.Node()-		missing := n.IsMissing()-		shown := missing || n.IsNamed()-		if shown {-			id := n.KindId()-			kind, ok := kinds[id]-			if !ok {-				kind = lang.NodeKindForId(id)-				kinds[id] = kind-			}-			if missing {-				kind = "MISSING " + kind-			}-			fid := cur.FieldId()-			field, ok := fields[fid]-			if !ok {-				field = lang.FieldNameForId(fid)-				fields[fid] = field-			}-			s, e := n.StartPosition(), n.EndPosition()-			w.WriteString(strconv.Itoa(len(named)))-			for _, f := range []string{kind, field} {-				w.WriteByte('\t')-				w.WriteString(f)-			}-			for _, v := range []uint{s.Row, s.Column, e.Row, e.Column} {-				w.WriteByte('\t')-				w.WriteString(strconv.FormatUint(uint64(v), 10))-			}-			w.WriteByte('\n')-		}-		if cur.GotoFirstChild() {-			if shown {-				named = append(named, depth)-			}-			depth++-			continue-		}-		for !cur.GotoNextSibling() {-			if !cur.GotoParent() {-				return-			}-			depth---			if len(named) > 0 && named[len(named)-1] == depth {-				named = named[:len(named)-1]-			}-		}+	// The NUL gives C a valid pointer for an empty source.+	buf := append(src, 0)+	switch code := syscall.Errno(C.syntax_tree(lang(), (*C.char)(unsafe.Pointer(&buf[0])), C.uint32_t(len(src)))); code {+	case 0:+	case syscall.EINVAL:+		os.Exit(fail(code, "cannot load the grammar: its ABI version does not match tree-sitter's"))+	case syscall.EIO:+		os.Exit(fail(code, "the parser gave no tree"))+	case syscall.ENOMEM:+		os.Exit(fail(code, "out of memory"))+	default:+		os.Exit(fail(code, "cannot write the tree: "+code.Error())) 	} } 

syntax/walk.c

new file mode 100644@@ -0,0 +1,219 @@+// walk.c parses the source and writes its named nodes in one call from Go.+// The Go binding crosses cgo for each node it is asked about, about ten+// times a node, anonymous nodes included, which took longer than the+// parse itself; here the walk never leaves C.++#include <errno.h>+#include <stdbool.h>+#include <stdint.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>++// The declarations below are copied from tree-sitter's api.h, as+// go-tree-sitter v0.25.0 ships it; cgo has no path to that header in the+// module cache. The layouts of TSPoint, TSNode and TSTreeCursor must+// match it, since they pass by value.+typedef uint16_t TSSymbol;+typedef uint16_t TSFieldId;+typedef struct TSLanguage TSLanguage;+typedef struct TSParser TSParser;+typedef struct TSTree TSTree;++typedef struct TSPoint {+	uint32_t row;+	uint32_t column;+} TSPoint;++typedef struct TSNode {+	uint32_t context[4];+	const void *id;+	const TSTree *tree;+} TSNode;++typedef struct TSTreeCursor {+	const void *tree;+	const void *id;+	uint32_t context[3];+} TSTreeCursor;++TSParser *ts_parser_new(void);+void ts_parser_delete(TSParser *self);+bool ts_parser_set_language(TSParser *self, const TSLanguage *language);+TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length);+void ts_tree_delete(TSTree *self);+TSNode ts_tree_root_node(const TSTree *self);+TSSymbol ts_node_symbol(TSNode self);+TSPoint ts_node_start_point(TSNode self);+TSPoint ts_node_end_point(TSNode self);+bool ts_node_is_named(TSNode self);+bool ts_node_is_missing(TSNode self);+TSTreeCursor ts_tree_cursor_new(TSNode node);+void ts_tree_cursor_delete(TSTreeCursor *self);+TSNode ts_tree_cursor_current_node(const TSTreeCursor *self);+TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *self);+bool ts_tree_cursor_goto_parent(TSTreeCursor *self);+bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self);+bool ts_tree_cursor_goto_first_child(TSTreeCursor *self);+const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol);+const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id);++// out buffers the rows for stdout. err holds the first failed write's+// errno, after which nothing more is written.+struct out {+	char buf[1 << 16];+	size_t n;+	int err;+};++static void flush(struct out *o) {+	size_t done = 0;+	while (done < o->n && o->err == 0) {+		ssize_t w = write(1, o->buf + done, o->n - done);+		if (w < 0) {+			if (errno != EINTR) {+				o->err = errno;+			}+			continue;+		}+		done += (size_t)w;+	}+	o->n = 0;+}++static void put(struct out *o, const char *s, size_t len) {+	if (o->n + len > sizeof o->buf) {+		flush(o);+	}+	// A kind longer than the buffer is not in any grammar, but writes+	// whole all the same.+	if (len > sizeof o->buf) {+		memcpy(o->buf, s, sizeof o->buf);+		o->n = sizeof o->buf;+		flush(o);+		put(o, s + sizeof o->buf, len - sizeof o->buf);+		return;+	}+	memcpy(o->buf + o->n, s, len);+	o->n += len;+}++static void putnum(struct out *o, uint64_t v) {+	char d[20];+	int i = sizeof d;+	do {+		d[--i] = '0' + v % 10;+		v /= 10;+	} while (v > 0);+	put(o, d + i, sizeof d - i);+}++static void puttab(struct out *o) {+	put(o, "\t", 1);+}++// syntax_tree parses src of len bytes by lang and writes the named nodes+// in preorder to stdout, one per line, as main.go's comment describes.+// Anonymous nodes, eg punctuation and keywords, are left out, but their+// named descendants keep their place under the nearest named ancestor.+// It returns 0, EINVAL when the grammar does not load, EIO when the+// parser gives no tree, ENOMEM, or the errno of a failed write.+int syntax_tree(const TSLanguage *lang, const char *src, uint32_t len) {+	TSParser *p = ts_parser_new();+	if (!ts_parser_set_language(p, lang)) {+		ts_parser_delete(p);+		return EINVAL;+	}+	TSTree *tree = ts_parser_parse_string(p, NULL, src, len);+	ts_parser_delete(p);+	if (tree == NULL) {+		return EIO;+	}+	struct out *o = calloc(1, sizeof *o);+	// named holds the cursor depths of the shown nodes above the cursor;+	// a row's depth is how many there are.+	size_t cap = 64, top = 0;+	uint32_t *named = malloc(cap * sizeof *named);+	if (o == NULL || named == NULL) {+		free(o);+		free(named);+		ts_tree_delete(tree);+		return ENOMEM;+	}+	int ret = 0;+	TSTreeCursor cur = ts_tree_cursor_new(ts_tree_root_node(tree));+	uint32_t depth = 0;+	for (;;) {+		TSNode n = ts_tree_cursor_current_node(&cur);+		bool missing = ts_node_is_missing(n);+		bool shown = missing || ts_node_is_named(n);+		if (shown) {+			const char *kind = ts_language_symbol_name(lang, ts_node_symbol(n));+			const char *field = ts_language_field_name_for_id(lang, ts_tree_cursor_current_field_id(&cur));+			TSPoint s = ts_node_start_point(n), e = ts_node_end_point(n);+			putnum(o, top);+			puttab(o);+			if (missing) {+				put(o, "MISSING ", 8);+			}+			if (kind != NULL) {+				put(o, kind, strlen(kind));+			}+			puttab(o);+			if (field != NULL) {+				put(o, field, strlen(field));+			}+			puttab(o);+			putnum(o, s.row);+			puttab(o);+			putnum(o, s.column);+			puttab(o);+			putnum(o, e.row);+			puttab(o);+			putnum(o, e.column);+			put(o, "\n", 1);+			if (o->err != 0) {+				break;+			}+		}+		if (ts_tree_cursor_goto_first_child(&cur)) {+			if (shown) {+				if (top == cap) {+					uint32_t *more = realloc(named, 2 * cap * sizeof *named);+					if (more == NULL) {+						ret = ENOMEM;+						break;+					}+					named = more;+					cap *= 2;+				}+				named[top++] = depth;+			}+			depth++;+			continue;+		}+		bool done = false;+		while (!ts_tree_cursor_goto_next_sibling(&cur)) {+			if (!ts_tree_cursor_goto_parent(&cur)) {+				done = true;+				break;+			}+			depth--;+			if (top > 0 && named[top - 1] == depth) {+				top--;+			}+		}+		if (done) {+			break;+		}+	}+	flush(o);+	if (ret == 0) {+		ret = o->err;+	}+	ts_tree_cursor_delete(&cur);+	ts_tree_delete(tree);+	free(named);+	free(o);+	return ret;+}