Changes to web.go. All files

web.go

@@ -1,9 +1,7 @@ package main  import (-	"bufio" 	"context"-	"encoding/json" 	"errors" 	"fmt" 	"html/template"@@ -15,7 +13,6 @@ import ( 	"os" 	"path" 	"path/filepath"-	"regexp" 	"slices" 	"strconv" 	"strings"@@ -589,624 +586,6 @@ func (w *web) run(j *webJob, key string, segs []string) { 	w.end(j, nil) } -// depths gives, for forges whose repos sit at one depth, the number of-// segments in a repo's key, host included.-var depths = map[string]int{"github.com": 3, "bitbucket.org": 3, "codeberg.org": 3}--// minDepths gives the fewest segments of a repo's key on forges with-// groups, host included.-var minDepths = map[string]int{"gitlab.com": 3}--// forgeViews gives, for each forge a doc's links are read from, the path-// segments after a repo's key that lead into a file or directory at a-// ref, eg blob and tree on GitHub, and raw for a file's bare bytes. The-// ref follows them, then the path.-var forgeViews = map[string][][]string{-	"github.com":    {{"blob"}, {"tree"}, {"raw"}},-	"codeberg.org":  {{"src", "branch"}, {"src", "tag"}, {"src", "commit"}, {"raw", "branch"}, {"raw", "tag"}, {"raw", "commit"}},-	"bitbucket.org": {{"src"}, {"raw"}},-	"gitlab.com":    {{"-", "blob"}, {"-", "tree"}, {"-", "raw"}},-}--// rawHosts serve a forge's files bare, at owner/name, then the ref, then-// the path, eg raw.githubusercontent.com/junegunn/fzf/master/README.md.-// Each maps to its forge.-var rawHosts = map[string]string{"raw.githubusercontent.com": "github.com"}--// pinnedRef matches a ref that is a commit's hash, which the web address-// serves as a version.-var pinnedRef = regexp.MustCompile(`^[0-9a-f]{7,40}$`)--// forgePages are GitHub's first segments that name a page of the site,-// not an account, eg github.com/sponsors/junegunn.-var forgePages = map[string]bool{-	"about": true, "apps": true, "collections": true, "customer-stories": true, "enterprise": true,-	"features": true, "login": true, "marketplace": true, "orgs": true, "settings": true,-	"site": true, "sponsors": true, "topics": true, "trending": true, "users": true,-}--// webPath is the path on the web address that reads what the forge URL u-// shows, eg /github.com/junegunn/fzf/README.md for-// https://github.com/junegunn/fzf/blob/master/README.md, and the repo's-// key, or "" when u is not a repo or a file in one on a forge in-// forgeViews or rawHosts. A ref that is a commit's hash leads to that-// version, eg /github.com/junegunn/fzf@26ff936/README.md; any other ref-// drops, since the web address serves the commit it fetched.-func webPath(u *url.URL) (key, p string) {-	if u.Scheme != "https" && u.Scheme != "http" || u.User != nil || u.Port() != "" {-		return "", ""-	}-	host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")-	views, ok := forgeViews[host]-	raw := rawHosts[host]-	if raw != "" {-		host, views, ok = raw, [][]string{{}}, true-	}-	if !ok {-		return "", ""-	}-	segs := strings.FieldsFunc(u.Path, func(r rune) bool { return r == '/' })-	if slices.ContainsFunc(segs, func(s string) bool { return s == "." || s == ".." }) {-		return "", ""-	}-	// A repo's key ends where a view begins, or after owner/name.-	n := len(segs)-	if host == "gitlab.com" {-		if i := slices.Index(segs, "-"); i >= 0 {-			n = i-		}-	} else {-		n = min(n, 2)-	}-	if n < 2 || host == "github.com" && forgePages[segs[0]] {-		return "", ""-	}-	k := append([]string{host}, segs[:n]...)-	k[n] = strings.TrimSuffix(k[n], ".git")-	key = strings.Join(k, "/")-	p = "/" + key + "/"-	rest := segs[n:]-	if len(rest) == 0 && raw == "" {-		return key, p-	}-	for _, v := range views {-		if len(rest) <= len(v) || !slices.Equal(rest[:len(v)], v) {-			continue-		}-		ref, after := rest[len(v)], rest[len(v)+1:]-		// raw.githubusercontent.com also names a ref in full, eg-		// refs/heads/main.-		if raw != "" && ref == "refs" && len(after) > 2 {-			after = after[2:]-		}-		file := strings.Join(after, "/")-		if raw != "" && file == "" {-			break-		}-		if file != "" && len(v) > 0 && v[len(v)-1] == "tree" {-			file += "/"-		}-		if pinnedRef.MatchString(ref) {-			p = "/" + key + "@" + ref + "/"-		}-		return key, p + file-	}-	return "", ""-}--// forgeStyle is how a forge links a file, a directory and a commit, each-// followed by a commit's hash, and how it marks lines in a fragment:-// github as #L10-L12, gitlab as #L10-12, bitbucket as #lines-10:12.-type forgeStyle struct{ file, dir, commit, lines string }--var forgeStyles = map[string]forgeStyle{-	"github.com":    {"/blob/", "/tree/", "/commit/", "github"},-	"codeberg.org":  {"/src/commit/", "/src/commit/", "/commit/", "github"},-	"gitlab.com":    {"/-/blob/", "/-/tree/", "/-/commit/", "gitlab"},-	"bitbucket.org": {"/src/", "/src/", "/commits/", "bitbucket"},-}--// forgeLink is the forge's own permanent URL for the page of kind that r-// asks of t, a repo on the web address or a version of one, and how the-// forge marks lines; "" where the forge or the page has none. A file or-// directory is linked at the commit the page shows, so the link keeps-// naming the same text after the forge moves on.-func (t *tree) forgeLink(r *http.Request, kind string) (link, lines string) {-	repo := t.web-	if t.base != nil {-		repo = t.base.web-	}-	if repo == nil {-		return "", ""-	}-	host, _, _ := strings.Cut(repo.key, "/")-	st, ok := forgeStyles[host]-	if !ok {-		return "", ""-	}-	base := "https://" + repo.key-	var commit, prefix string-	switch kind {-	case "commit":-		if h := r.URL.Query().Get("commit"); hexHash.MatchString(h) {-			return base + st.commit + h, st.lines-		}-		return "", ""-	case "doc", "code", "tree":-		prefix = st.file-	case "dir":-		prefix = st.dir-	default:-		return "", ""-	}-	if t.at != nil {-		commit = t.at.hash-	} else if commit = repoAt(r.Context(), t.dir).head(r.Context()); !isHash(commit) {-		return "", ""-	}-	link = base + prefix + commit-	if name := t.name(r.URL.Path); name != "." {-		link += (&url.URL{Path: "/" + name}).EscapedPath()-	}-	return link, st.lines-}--// probe asks every prefix of the path at once whether a repo answers-// there, each with and without .git, and takes the longest that does. It-// returns the repo's key and URL, or "" and whether it probed at all.-func (w *web) probe(j *webJob, segs []string) (key, cloneURL string, probed bool) {-	lo, hi := 2, min(len(segs), probeMax+1)-	if d, ok := depths[segs[0]]; ok {-		// The repo is the first d segments, so a probe would tell-		// nothing the clone does not, and costs a round trip.-		if len(segs) >= d {-			key = strings.Join(segs[:d], "/")-			w.step(j, segs[0]+" keeps every repo at owner/name, so "+webScheme+"://"+key+" needs no probe")-			return key, webScheme + "://" + key, true-		}-		lo, hi = d, min(d, len(segs))-	} else if d, ok := minDepths[segs[0]]; ok {-		lo = d-	}-	if lo > hi {-		return "", "", false-	}-	type answer struct {-		n   int-		url string-	}-	var urls []string-	for n := hi; n >= lo; n-- {-		u := webScheme + "://" + strings.Join(segs[:n], "/")-		urls = append(urls, u)-	}-	w.step(j, "git ls-remote, at once, on "+strings.Join(urls, " "))-	start := time.Now()-	ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second)-	defer cancel()-	ch := make(chan answer)-	for n := hi; n >= lo; n-- {-		for _, u := range []string{webScheme + "://" + strings.Join(segs[:n], "/"), webScheme + "://" + strings.Join(segs[:n], "/") + ".git"} {-			go func() {-				if _, err := git(ctx, w.root, "ls-remote", u, "HEAD"); err == nil {-					ch <- answer{n, u}-				} else {-					ch <- answer{0, ""}-				}-			}()-		}-	}-	var best answer-	for range 2 * (hi - lo + 1) {-		if a := <-ch; a.n > best.n || a.n == best.n && a.n > 0 && len(a.url) < len(best.url) {-			best = a-		}-	}-	ms := time.Since(start).Milliseconds()-	if best.n == 0 {-		w.step(j, fmt.Sprintf("no repo answered, in %d ms", ms))-		return "", "", true-	}-	w.step(j, fmt.Sprintf("%s answered, in %d ms", best.url, ms))-	return strings.Join(segs[:best.n], "/"), best.url, true-}--// clone fetches the repo key from cloneURL into the cache and serves it.-func (w *web) clone(j *webJob, key, cloneURL string) error {-	w.mu.Lock()-	r := w.repos[key]-	w.mu.Unlock()-	if r != nil {-		return nil-	}-	dir := filepath.Join(w.root, filepath.FromSlash(key))-	if err := os.MkdirAll(dir, 0o700); err != nil {-		return err-	}-	unlock, err := w.lockRepo(key, dir)-	if err != nil {-		return err-	}-	defer unlock()-	// Another visit or server may have cloned it meanwhile.-	if r, err := w.load(key, true); r != nil || err != nil {-		return err-	}-	bare := filepath.Join(dir, ".git")-	if !exists(filepath.Join(bare, "HEAD")) {-		if w.slots != nil {-			select {-			case w.slots <- struct{}{}:-			default:-				w.step(j, fmt.Sprintf("waiting: %d clones run at once", cap(w.slots)))-				select {-				case w.slots <- struct{}{}:-				case <-w.ctx.Done():-					return w.ctx.Err()-				}-			}-			// The slot covers the worktree's blobs too, the larger fetch.-			defer func() { <-w.slots }()-		}-		tmp := bare + ".tmp"-		os.RemoveAll(tmp)-		w.step(j, "git clone --bare --filter=blob:none "+cloneURL+" "+w.where(bare))-		start := time.Now()-		ctx, stop := w.bounded(dir)-		_, err := gitProgress(ctx, dir, w.progress(j), "clone", "--bare", "--filter=blob:none", "--progress", "--", cloneURL, tmp)-		err = boundErr(ctx, err)-		stop()-		if err != nil {-			os.RemoveAll(tmp)-			return err-		}-		if err := os.Rename(tmp, bare); err != nil {-			return err-		}-		w.step(j, fmt.Sprintf("cloned in %s", seconds(time.Since(start))))-	}-	r = &webRepo{key: key, dir: dir}-	touch(filepath.Join(dir, ".fetched"))-	touch(filepath.Join(dir, ".used"))-	if _, err := w.checkout(r, w.progress(j), func(s string) { w.step(j, s) }); err != nil {-		return err-	}-	// The cache must hold the new repo before a page shows it.-	if err := w.trim(key); err != nil {-		return err-	}-	_, err = w.load(key, true)-	return err-}--// checkout builds the worktree of the commit HEAD names, unless it is-// there, and points .tree at it.-func (w *web) checkout(r *webRepo, progress func(string), step func(string)) (string, error) {-	bare := filepath.Join(r.dir, ".git")-	out, err := git(w.ctx, bare, "rev-parse", "--verify", "HEAD^{commit}")-	if err != nil {-		return "", err-	}-	head := strings.TrimSpace(string(out))-	name := ".wt-" + head[:12]-	wt := filepath.Join(r.dir, name)-	if !exists(wt) {-		step("git worktree add --detach " + w.where(wt) + " " + head[:12] + ", which fetches its files")-		start := time.Now()-		ctx, stop := w.bounded(r.dir)-		_, err := gitProgress(ctx, bare, progress, "worktree", "add", "--detach", "--", wt, head)-		err = boundErr(ctx, err)-		stop()-		if err != nil {-			os.RemoveAll(wt)-			git(w.ctx, bare, "worktree", "prune")-			return "", err-		}-		step(fmt.Sprintf("checked out in %s", seconds(time.Since(start))))-	}-	link := filepath.Join(r.dir, ".tree")-	os.Remove(link + ".new")-	if err := os.Symlink(name, link+".new"); err != nil {-		return "", err-	}-	return wt, os.Rename(link+".new", link)-}--// refresh fetches r again, and serves the new commit when HEAD moved. A-// failed fetch leaves the commit served, and its pages say so.-func (w *web) refresh(r *webRepo) {-	failed := ""-	defer func() {-		r.mu.Lock()-		r.fetching, r.failed = false, failed-		if failed == "" {-			r.fetched = time.Now()-		}-		r.mu.Unlock()-		w.kick(true)-	}()-	w.kick(true)-	unlock, err := w.lockRepo(r.key, r.dir)-	if err != nil {-		failed = err.Error()-		return-	}-	defer unlock()-	bare := filepath.Join(r.dir, ".git")-	start := time.Now()-	ctx, stop := w.bounded(r.dir)-	_, err = git(ctx, bare, "fetch", "--prune", "--no-write-fetch-head", "origin",-		"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")-	err = boundErr(ctx, err)-	stop()-	if err != nil {-		failed = err.Error()-		logErr("E%d: cannot fetch %s: %s", syscall.EHOSTUNREACH, r.key, failed)-		return-	}-	touch(filepath.Join(r.dir, ".fetched"))-	r.mu.Lock()-	old := r.tree.dir-	r.mu.Unlock()-	wt, err := w.checkout(r, nil, func(string) {})-	if err != nil {-		failed = err.Error()-		return-	}-	logInfo("fetched %s in %d ms", r.key, time.Since(start).Milliseconds())-	if wt == old {-		return-	}-	if err := w.show(r, wt); err != nil {-		failed = err.Error()-		return-	}-	w.dropTree(r, old)-	if err := w.trim(r.key); err != nil {-		failed = err.Error()-	}-}--// fetchWait bounds a public server's clone, fetch or checkout, so a forge-// that sends slowly cannot hold a clone's place for good.-const fetchWait = 10 * time.Minute--// bounded is the context for git fetching into the repo directory dir.-// On a public server it ends after fetchWait, or once dir outgrows the-// cache's limit: trim runs only after a fetch, and a stranger's forge may-// send without end.-func (w *web) bounded(dir string) (context.Context, func()) {-	if !w.public {-		return w.ctx, func() {}-	}-	ctx, cancel := context.WithCancelCause(w.ctx)-	ctx, cancelWait := context.WithTimeoutCause(ctx, fetchWait,-		&errno{syscall.ETIMEDOUT, fmt.Sprintf("the forge took over %d minutes, the most a public server waits", int(fetchWait.Minutes()))})-	done := make(chan struct{})-	w.wg.Go(func() {-		tick := time.NewTicker(2 * time.Second)-		defer tick.Stop()-		for {-			select {-			case <-done:-				return-			case <-tick.C:-				if n := diskUse(dir); n > w.limit {-					cancel(&errno{syscall.EFBIG, fmt.Sprintf("the repo grew past %s, the cache's limit, so the fetch stopped; --disk sets the limit", sizeText(w.limit))})-					return-				}-			}-		}-	})-	return ctx, func() {-		close(done)-		cancelWait()-		cancel(nil)-	}-}--// boundErr is why git failed in a context from bounded: the limit it hit,-// or err.-func boundErr(ctx context.Context, err error) error {-	if err != nil && ctx.Err() != nil {-		if cause := context.Cause(ctx); cause != nil {-			var e *errno-			if errors.As(cause, &e) {-				return cause-			}-		}-	}-	return err-}--// lockRepo marks the repo key busy and takes its lock.-func (w *web) lockRepo(key, dir string) (func(), error) {-	w.mu.Lock()-	w.busy[key] = true-	w.mu.Unlock()-	unlock, err := flockDir(dir, syscall.LOCK_EX)-	if err != nil {-		unlock = func() {}-	}-	return func() {-		unlock()-		w.mu.Lock()-		delete(w.busy, key)-		w.mu.Unlock()-	}, err-}--// flockDir takes the repo directory's lock, which keeps two servers from-// fetching one repo at once, and one from deleting a repo another fetches.-// how is LOCK_EX, or LOCK_EX|LOCK_NB to fail with EWOULDBLOCK when held.-func flockDir(dir string, how int) (func(), error) {-	f, err := os.OpenFile(filepath.Join(dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600)-	if err != nil {-		return nil, err-	}-	if err := flock(f, how); err != nil {-		f.Close()-		return nil, err-	}-	return func() { f.Close() }, nil-}--// cachedRepo is a repo found on disk in the cache.-type cachedRepo struct {-	key  string-	size int64-	used time.Time-}--// onDisk lists the repos in the cache, with their size and last request.-func (w *web) onDisk() []cachedRepo {-	var out []cachedRepo-	filepath.WalkDir(w.root, func(p string, d fs.DirEntry, err error) error {-		if err != nil || !d.IsDir() {-			return nil-		}-		name := d.Name()-		if p != w.root && (strings.HasPrefix(name, ".wt-") || name == ".git" || name == ".git.tmp") {-			return fs.SkipDir-		}-		if !exists(filepath.Join(p, ".git", "HEAD")) {-			return nil-		}-		key, _ := filepath.Rel(w.root, p)-		c := cachedRepo{key: filepath.ToSlash(key), size: diskUse(filepath.Join(p, ".git"))}-		if wt, err := os.Readlink(filepath.Join(p, ".tree")); err == nil {-			c.size += diskUse(filepath.Join(p, wt))-		}-		for _, f := range []string{".used", ".fetched"} {-			if info, err := os.Stat(filepath.Join(p, f)); err == nil {-				c.used = info.ModTime()-				break-			}-		}-		out = append(out, c)-		return nil-	})-	return out-}--// diskUse is the disk space below dir.-func diskUse(dir string) int64 {-	var n int64-	filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {-		if err != nil {-			return nil-		}-		if info, err := d.Info(); err == nil {-			if st, ok := info.Sys().(*syscall.Stat_t); ok {-				n += st.Blocks * 512-			}-		}-		return nil-	})-	return n-}--// trim drops the repos requested longest ago until the cache fits its-// limit. A repo in use, fetched here or locked by another server, stays.-// keep, just fetched, goes last, and only when it alone is over the limit;-// trim then says so.-func (w *web) trim(keep string) error {-	repos := w.onDisk()-	var total int64-	for _, c := range repos {-		total += c.size-	}-	slices.SortFunc(repos, func(a, b cachedRepo) int {-		if (a.key == keep) != (b.key == keep) {-			if a.key == keep {-				return 1-			}-			return -1-		}-		return a.used.Compare(b.used)-	})-	for _, c := range repos {-		if total <= w.limit {-			return nil-		}-		w.mu.Lock()-		busy := w.busy[c.key] && c.key != keep-		w.mu.Unlock()-		if busy {-			continue-		}-		if c.key == keep && c.size <= w.limit {-			// Repos in use kept the cache over its limit; a later trim-			// drops them.-			logInfo("the cache stays over %s while repos in use fill it", sizeText(w.limit))-			return nil-		}-		if c.key == keep {-			// The caller holds keep's lock.-			w.remove(c.key)-			return &errno{syscall.EFBIG, fmt.Sprintf("%s takes %s, over the cache's limit of %s, so it was dropped; --disk sets the limit",-				c.key, sizeText(c.size), sizeText(w.limit))}-		}-		if !w.drop(c.key) {-			logInfo("keeps %s, %s, though the cache is over %s, since another server is fetching it", c.key, sizeText(c.size), sizeText(w.limit))-			continue-		}-		logInfo("dropped %s, %s, last requested %s, since the cache is over %s", c.key, sizeText(c.size), c.used.Format(time.DateTime), sizeText(w.limit))-		total -= c.size-	}-	return nil-}--// drop stops serving the repo key and deletes it from the cache, unless-// another server holds its lock, and reports whether it did. The servers-// on every address share the cache, and each knows only its own busy-// repos.-func (w *web) drop(key string) bool {-	dir := filepath.Join(w.root, filepath.FromSlash(key))-	unlock, err := flockDir(dir, syscall.LOCK_EX|syscall.LOCK_NB)-	if errors.Is(err, syscall.EWOULDBLOCK) {-		return false-	}-	if err == nil {-		defer unlock()-	}-	w.remove(key)-	return true-}--// remove stops serving the repo key and deletes it from the cache. The-// caller holds its lock.-func (w *web) remove(key string) {-	w.mu.Lock()-	r := w.repos[key]-	w.mu.Unlock()-	w.forget(r)-	dir := filepath.Join(w.root, filepath.FromSlash(key))-	os.RemoveAll(dir)-	for d := filepath.Dir(dir); d != w.root && strings.HasPrefix(d, w.root); d = filepath.Dir(d) {-		if os.Remove(d) != nil {-			break-		}-	}-}--// forget stops serving r and leaves the disk as it is.-func (w *web) forget(r *webRepo) {-	if r == nil {-		return-	}-	w.mu.Lock()-	if w.repos[r.key] == r {-		delete(w.repos, r.key)-	}-	w.mu.Unlock()-	r.mu.Lock()-	t := r.tree-	r.mu.Unlock()-	w.srv.remove(t)-	t.root.Close()-}- func sizeText(n int64) string { 	switch { 	case n >= 1<<30:@@ -1248,203 +627,6 @@ func tildePath(p string) string { 	return p } -// listing is the repos a host or account holds, and what told.-type listing struct {-	at     time.Time-	source string   // what answered, eg "the GitHub API"-	asked  string   // the URL asked-	failed []string // the sources that failed before it, and why-	repos  []listed-}--type listed struct{ key, desc string }--func (l *listing) fresh() time.Duration {-	if len(l.repos) == 0 {-		return missFresh-	}-	return listFresh-}--// list lists the path key, by the first source that answers: the forge's-// API, a web frontend's index, then the cache.-func (w *web) list(j *webJob, key string, probed bool) {-	segs := strings.Split(key, "/")-	l := &listing{at: time.Now()}-	for _, src := range listSources(segs) {-		w.step(j, "GET "+src.url+", "+src.name)-		repos, err := w.ask(src)-		if err == nil && len(repos) > 0 {-			l.source, l.asked, l.repos = src.name, src.url, repos-			break-		}-		if err == nil {-			err = errors.New("no repos")-		}-		l.failed = append(l.failed, src.name+": "+err.Error())-		w.step(j, src.name+": "+err.Error())-	}-	// A stranger must not learn what others read, so a public server-	// never lists the cache.-	if l.source == "" && !w.public {-		l.source = "the cache"-		for _, c := range w.onDisk() {-			if strings.HasPrefix(c.key, key+"/") {-				l.repos = append(l.repos, listed{key: c.key})-			}-		}-		slices.SortFunc(l.repos, func(a, b listed) int { return strings.Compare(a.key, b.key) })-	}-	if len(l.repos) == 0 && probed && len(l.failed) == 0 {-		l.failed = append(l.failed, "no source lists "+key)-	}-	logInfo("listed %d repos at %s by %s", len(l.repos), key, l.source)-	w.mu.Lock()-	w.lists[key] = l-	w.mu.Unlock()-	w.end(j, nil)-}--// listSource is one place that may list a path's repos.-type listSource struct {-	name, url string-	parse     func(body []byte, next *string) ([]listed, error)-}--var (-	cgitRepo   = regexp.MustCompile(`<td class='(?:top|sub)level-repo'><a (?:title='[^']*' )?href='([^']+)'`)-	stagitRepo = regexp.MustCompile(`<td><a href="([^"/]+)/log\.html">`)-)--// listSources are the sources for the path segs, in the order to ask them.-func listSources(segs []string) []listSource {-	host, rest := segs[0], strings.Join(segs[1:], "/")-	var out []listSource-	jsonList := func(field string) func([]byte, *string) ([]listed, error) {-		return func(b []byte, _ *string) ([]listed, error) {-			var v []map[string]any-			if err := json.Unmarshal(b, &v); err != nil {-				return nil, errors.New("not a list of repos")-			}-			var out []listed-			for _, m := range v {-				name, _ := m[field].(string)-				desc, _ := m["description"].(string)-				if name != "" {-					out = append(out, listed{key: host + "/" + name, desc: desc})-				}-			}-			return out, nil-		}-	}-	switch {-	case host == "github.com" && len(segs) == 2:-		out = append(out, listSource{"the GitHub API", "https://api.github.com/users/" + rest + "/repos?per_page=100", jsonList("full_name")})-	case host == "gitlab.com" && len(segs) >= 2:-		if len(segs) == 2 {-			out = append(out, listSource{"the GitLab API", "https://gitlab.com/api/v4/users/" + rest + "/projects?per_page=100", jsonList("path_with_namespace")})-		}-		out = append(out, listSource{"the GitLab API", "https://gitlab.com/api/v4/groups/" + url.PathEscape(rest) + "/projects?per_page=100&include_subgroups=true", jsonList("path_with_namespace")})-	case host == "bitbucket.org" && len(segs) == 2:-		out = append(out, listSource{"the Bitbucket API", "https://api.bitbucket.org/2.0/repositories/" + rest + "?pagelen=100",-			func(b []byte, next *string) ([]listed, error) {-				var v struct {-					Values []struct{ Full_name, Description string }-					Next   string-				}-				if err := json.Unmarshal(b, &v); err != nil {-					return nil, errors.New("not a list of repos")-				}-				*next = v.Next-				var out []listed-				for _, r := range v.Values {-					out = append(out, listed{key: host + "/" + r.Full_name, desc: r.Description})-				}-				return out, nil-			}})-	case len(segs) == 2 && !slices.Contains([]string{"github.com", "gitlab.com", "bitbucket.org"}, host):-		out = append(out, listSource{"the Forgejo API", webScheme + "://" + host + "/api/v1/users/" + rest + "/repos?limit=50", jsonList("full_name")})-	}-	if _, forge := depths[host]; forge || host == "gitlab.com" {-		return out-	}-	base := webScheme + "://" + strings.Join(segs, "/") + "/"-	out = append(out, listSource{"gitweb's project index", base + "?a=project_index", func(b []byte, _ *string) ([]listed, error) {-		var out []listed-		sc := bufio.NewScanner(strings.NewReader(string(b)))-		for sc.Scan() {-			f := strings.Fields(sc.Text())-			if len(f) == 0 || strings.ContainsAny(f[0], "<>\"") {-				return nil, errors.New("not a project index")-			}-			name, err := url.QueryUnescape(f[0])-			if err != nil {-				return nil, errors.New("not a project index")-			}-			out = append(out, listed{key: strings.Join(segs, "/") + "/" + strings.TrimSuffix(name, ".git")})-		}-		return out, nil-	}})-	out = append(out, listSource{"the frontend's front page", base, func(b []byte, _ *string) ([]listed, error) {-		var out []listed-		for _, m := range cgitRepo.FindAllSubmatch(b, -1) {-			p := strings.Trim(string(m[1]), "/")-			out = append(out, listed{key: host + "/" + strings.TrimSuffix(p, ".git")})-		}-		for _, m := range stagitRepo.FindAllSubmatch(b, -1) {-			out = append(out, listed{key: strings.Join(segs, "/") + "/" + string(m[1])})-		}-		return out, nil-	}})-	return out-}--var webClient = &http.Client{Timeout: 20 * time.Second}--// ask reads src, following the forge's pages up to 10.-func (w *web) ask(src listSource) ([]listed, error) {-	var out []listed-	u := src.url-	for page := 1; page <= 10 && u != ""; page++ {-		req, err := http.NewRequestWithContext(w.ctx, "GET", u, nil)-		if err != nil {-			return nil, err-		}-		req.Header.Set("User-Agent", progName)-		client := webClient-		if w.public {-			client = publicClient-		}-		resp, err := client.Do(req)-		if err != nil {-			return nil, err-		}-		b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))-		resp.Body.Close()-		if err != nil {-			return nil, err-		}-		if resp.StatusCode != http.StatusOK {-			return nil, errors.New(resp.Status)-		}-		next := ""-		got, err := src.parse(b, &next)-		if err != nil {-			return nil, err-		}-		out = append(out, got...)-		switch {-		case next != "":-			u = next-		case strings.Contains(src.url, "api") && len(got) >= 50 && !strings.Contains(src.url, "bitbucket"):-			u = src.url + "&page=" + strconv.Itoa(page+1)-		default:-			u = ""-		}-	}-	return out, nil-}- // version writes the state of a page at urlPath that no tree serves. func (w *web) version(h io.Writer, urlPath string) { 	key := strings.Trim(path.Clean(urlPath), "/")@@ -1463,142 +645,3 @@ func (w *web) version(h io.Writer, urlPath string) { 	slices.Sort(keys) 	io.WriteString(h, strings.Join(keys, "\n")) }--// crumbs link the root and each segment of a path that no tree serves.-func webCrumbs(key string) []crumb {-	out := []crumb{indexCrumb}-	segs := strings.Split(key, "/")-	for i, s := range segs {-		c := crumb{Label: s}-		if i < len(segs)-1 {-			c.Href = (&url.URL{Path: "/" + strings.Join(segs[:i+1], "/") + "/"}).String()-		}-		out = append(out, c)-	}-	return out-}--func (w *web) render(rw http.ResponseWriter, r *http.Request, status int, title string, crumbs []crumb, kind string, body string) {-	render(rw, r, status, page{Title: title, Crumbs: crumbs, Kind: kind, Body: template.HTML(body), NoStop: true})-}--func (w *web) fail(rw http.ResponseWriter, r *http.Request, status int, msg string) {-	w.render(rw, r, status, "not found", []crumb{indexCrumb, {Label: "not found"}}, "error",-		"<pre class=\"error\">"+template.HTMLEscapeString(msg)+"</pre>")-}--// serveRoot lists the cache: each repo, its size and its last request. A-// public server says only how to read a repo.-func (w *web) serveRoot(rw http.ResponseWriter, r *http.Request) {-	esc := template.HTMLEscapeString-	repos := w.onDisk()-	slices.SortFunc(repos, func(a, b cachedRepo) int { return strings.Compare(a.key, b.key) })-	var total int64-	var body strings.Builder-	body.WriteString("<p class=\"own\">Type a forge's host and a path after this address, eg <a href=\"/github.com/junegunn/fzf/\">/github.com/junegunn/fzf</a>, to read that repo, or <a href=\"/git.suckless.org/\">/git.suckless.org</a> to list a host's repos.</p>\n")-	if w.public {-		w.render(rw, r, http.StatusOK, "web", []crumb{{Label: "index"}}, "index", body.String())-		return-	}-	if len(repos) > 0 {-		var sizes []string-		for _, c := range repos {-			sizes = append(sizes, sizeText(c.size))-		}-		// The size leads, aligned right as numbers are; when it was-		// read follows the name.-		body.WriteString("<table class=\"locks own\"" + leadStyle(sizes) + ">\n")-		for i, c := range repos {-			total += c.size-			fmt.Fprintf(&body, "<tr><td><span class=\"since lead num\">%s</span> <a href=\"%s\">%s</a> <span class=\"since\">last read %s</span></td></tr>\n",-				sizes[i], esc((&url.URL{Path: "/" + c.key + "/"}).String()), segments(c.key), clock(c.used))-		}-		body.WriteString("</table>\n")-	}-	fmt.Fprintf(&body, "<p class=\"own\">The cache in %s holds %d repos in %s of its %s.</p>\n", esc(tildePath(w.root)), len(repos), sizeText(total), sizeText(w.limit))-	if u := w.srv.homeURL; u != "" {-		fmt.Fprintf(&body, "<p class=\"own\">Trees: <a href=\"%s/\">%s/</a></p>\n", esc(u), esc(strings.TrimPrefix(u, "http://")))-	}-	w.render(rw, r, http.StatusOK, "web", []crumb{{Label: "index"}}, "index", body.String())-}--// serveJob shows what a visit set going: the reading page.-func (w *web) serveJob(rw http.ResponseWriter, r *http.Request, j *webJob) {-	esc := template.HTMLEscapeString-	w.mu.Lock()-	steps, line, failed, started := slices.Clone(j.steps), j.line, j.err, j.started-	w.mu.Unlock()-	var body strings.Builder-	status := http.StatusAccepted-	if failed != "" {-		status = http.StatusBadGateway-		body.WriteString("<pre class=\"error\">" + esc(failed) + "</pre>\n")-	} else {-		body.WriteString("<p class=\"own reading\">reading</p>\n")-	}-	body.WriteString("<pre class=\"own steps\">")-	for _, s := range steps {-		body.WriteString(esc(s) + "\n")-	}-	if line != "" {-		body.WriteString(esc(line) + "\n")-	}-	body.WriteString("</pre>\n")-	fmt.Fprintf(&body, "<p class=\"own\">Started %s, %d s ago.", clock(started), int(time.Since(started).Seconds()))-	if failed != "" {-		body.WriteString(" Reload after 30 s to try again.")-	} else {-		body.WriteString(" The page turns into the repo when it is read.")-	}-	body.WriteString("</p>\n")-	key := strings.TrimPrefix(j.urlPath, "/")-	w.render(rw, r, status, "reading "+key, webCrumbs(key), "reading", body.String())-}--// serveConfirm asks before a download that another site set going.-func (w *web) serveConfirm(rw http.ResponseWriter, r *http.Request, key string) {-	esc := template.HTMLEscapeString-	u := (&url.URL{Path: "/" + key}).String()-	body := fmt.Sprintf("<p class=\"own\">Another site sent you here, so nothing is fetched yet. <a href=\"%s\">Read %s</a> fetches it from %s into the cache.</p>\n",-		esc(u), esc(key), esc(webScheme+"://"+key))-	// The page runs the script, which carries the link's fragment, eg-	// #L10-L12, onto the read link, but holds no stream: a reload must-	// not fetch what the user has not asked for.-	render(rw, r, http.StatusOK, page{Title: key, Crumbs: webCrumbs(key), Kind: "confirm", Body: template.HTML(body), NoStop: true, Live: "-", Fixed: true})-}--// serveListing lists a host's or account's repos, and names its source.-func (w *web) serveListing(rw http.ResponseWriter, r *http.Request, l *listing) {-	esc := template.HTMLEscapeString-	key := strings.Trim(path.Clean(r.URL.Path), "/")-	var body strings.Builder-	status := http.StatusOK-	if len(l.repos) == 0 {-		status = http.StatusNotFound-		fmt.Fprintf(&body, "<pre class=\"error\">No repo answers at %s, and nothing lists one there.</pre>\n", esc(webScheme+"://"+key))-	} else {-		body.WriteString("<ul class=\"listing\">\n")-		for _, e := range l.repos {-			name := strings.TrimPrefix(e.key, key+"/")-			fmt.Fprintf(&body, "<li><a href=\"%s\">%s/</a>", esc((&url.URL{Path: "/" + e.key + "/"}).String()), esc(name))-			if e.desc != "" {-				fmt.Fprintf(&body, " <span class=\"own desc\">%s</span>", esc(e.desc))-			}-			body.WriteString("</li>\n")-		}-		body.WriteString("</ul>\n")-	}-	src := l.source-	if l.asked != "" {-		src += ", " + l.asked-	}-	fmt.Fprintf(&body, "<p class=\"own\">%d repos, listed by %s at %s.", len(l.repos), esc(src), clock(l.at))-	if l.source == "the cache" {-		body.WriteString(" The cache holds only what was read here, so the host may hold more.")-	}-	body.WriteString("</p>\n")-	if len(l.failed) > 0 {-		body.WriteString("<p class=\"own\">Failed first: " + esc(strings.Join(l.failed, "; ")) + ".</p>\n")-	}-	w.render(rw, r, status, key, webCrumbs(key), "index", body.String())-}