Changes to webclone.go. All files

webclone.go

new file mode 100644@@ -0,0 +1,312 @@+package main++import (+	"context"+	"errors"+	"fmt"+	"os"+	"path/filepath"+	"strings"+	"syscall"+	"time"+)++// probe asks every prefix of the path at once whether a repo answers+// there, each with and without .git, and takes the longest that does. It+// returns the repo's key and URL, or "" and whether it probed at all.+func (w *web) probe(j *webJob, segs []string) (key, cloneURL string, probed bool) {+	lo, hi := 2, min(len(segs), probeMax+1)+	if d, ok := depths[segs[0]]; ok {+		// The repo is the first d segments, so a probe would tell+		// nothing the clone does not, and costs a round trip.+		if len(segs) >= d {+			key = strings.Join(segs[:d], "/")+			w.step(j, segs[0]+" keeps every repo at owner/name, so "+webScheme+"://"+key+" needs no probe")+			return key, webScheme + "://" + key, true+		}+		lo, hi = d, min(d, len(segs))+	} else if d, ok := minDepths[segs[0]]; ok {+		lo = d+	}+	if lo > hi {+		return "", "", false+	}+	type answer struct {+		n   int+		url string+	}+	var urls []string+	for n := hi; n >= lo; n-- {+		u := webScheme + "://" + strings.Join(segs[:n], "/")+		urls = append(urls, u)+	}+	w.step(j, "git ls-remote, at once, on "+strings.Join(urls, " "))+	start := time.Now()+	ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second)+	defer cancel()+	ch := make(chan answer)+	for n := hi; n >= lo; n-- {+		for _, u := range []string{webScheme + "://" + strings.Join(segs[:n], "/"), webScheme + "://" + strings.Join(segs[:n], "/") + ".git"} {+			go func() {+				if _, err := git(ctx, w.root, "ls-remote", u, "HEAD"); err == nil {+					ch <- answer{n, u}+				} else {+					ch <- answer{0, ""}+				}+			}()+		}+	}+	var best answer+	for range 2 * (hi - lo + 1) {+		if a := <-ch; a.n > best.n || a.n == best.n && a.n > 0 && len(a.url) < len(best.url) {+			best = a+		}+	}+	ms := time.Since(start).Milliseconds()+	if best.n == 0 {+		w.step(j, fmt.Sprintf("no repo answered, in %d ms", ms))+		return "", "", true+	}+	w.step(j, fmt.Sprintf("%s answered, in %d ms", best.url, ms))+	return strings.Join(segs[:best.n], "/"), best.url, true+}++// clone fetches the repo key from cloneURL into the cache and serves it.+func (w *web) clone(j *webJob, key, cloneURL string) error {+	w.mu.Lock()+	r := w.repos[key]+	w.mu.Unlock()+	if r != nil {+		return nil+	}+	dir := filepath.Join(w.root, filepath.FromSlash(key))+	if err := os.MkdirAll(dir, 0o700); err != nil {+		return err+	}+	unlock, err := w.lockRepo(key, dir)+	if err != nil {+		return err+	}+	defer unlock()+	// Another visit or server may have cloned it meanwhile.+	if r, err := w.load(key, true); r != nil || err != nil {+		return err+	}+	bare := filepath.Join(dir, ".git")+	if !exists(filepath.Join(bare, "HEAD")) {+		if w.slots != nil {+			select {+			case w.slots <- struct{}{}:+			default:+				w.step(j, fmt.Sprintf("waiting: %d clones run at once", cap(w.slots)))+				select {+				case w.slots <- struct{}{}:+				case <-w.ctx.Done():+					return w.ctx.Err()+				}+			}+			// The slot covers the worktree's blobs too, the larger fetch.+			defer func() { <-w.slots }()+		}+		tmp := bare + ".tmp"+		os.RemoveAll(tmp)+		w.step(j, "git clone --bare --filter=blob:none "+cloneURL+" "+w.where(bare))+		start := time.Now()+		ctx, stop := w.bounded(dir)+		_, err := gitProgress(ctx, dir, w.progress(j), "clone", "--bare", "--filter=blob:none", "--progress", "--", cloneURL, tmp)+		err = boundErr(ctx, err)+		stop()+		if err != nil {+			os.RemoveAll(tmp)+			return err+		}+		if err := os.Rename(tmp, bare); err != nil {+			return err+		}+		w.step(j, fmt.Sprintf("cloned in %s", seconds(time.Since(start))))+	}+	r = &webRepo{key: key, dir: dir}+	touch(filepath.Join(dir, ".fetched"))+	touch(filepath.Join(dir, ".used"))+	if _, err := w.checkout(r, w.progress(j), func(s string) { w.step(j, s) }); err != nil {+		return err+	}+	// The cache must hold the new repo before a page shows it.+	if err := w.trim(key); err != nil {+		return err+	}+	_, err = w.load(key, true)+	return err+}++// checkout builds the worktree of the commit HEAD names, unless it is+// there, and points .tree at it.+func (w *web) checkout(r *webRepo, progress func(string), step func(string)) (string, error) {+	bare := filepath.Join(r.dir, ".git")+	out, err := git(w.ctx, bare, "rev-parse", "--verify", "HEAD^{commit}")+	if err != nil {+		return "", err+	}+	head := strings.TrimSpace(string(out))+	name := ".wt-" + head[:12]+	wt := filepath.Join(r.dir, name)+	if !exists(wt) {+		step("git worktree add --detach " + w.where(wt) + " " + head[:12] + ", which fetches its files")+		start := time.Now()+		ctx, stop := w.bounded(r.dir)+		_, err := gitProgress(ctx, bare, progress, "worktree", "add", "--detach", "--", wt, head)+		err = boundErr(ctx, err)+		stop()+		if err != nil {+			os.RemoveAll(wt)+			git(w.ctx, bare, "worktree", "prune")+			return "", err+		}+		step(fmt.Sprintf("checked out in %s", seconds(time.Since(start))))+	}+	link := filepath.Join(r.dir, ".tree")+	os.Remove(link + ".new")+	if err := os.Symlink(name, link+".new"); err != nil {+		return "", err+	}+	return wt, os.Rename(link+".new", link)+}++// refresh fetches r again, and serves the new commit when HEAD moved. A+// failed fetch leaves the commit served, and its pages say so.+func (w *web) refresh(r *webRepo) {+	failed := ""+	defer func() {+		r.mu.Lock()+		r.fetching, r.failed = false, failed+		if failed == "" {+			r.fetched = time.Now()+		}+		r.mu.Unlock()+		w.kick(true)+	}()+	w.kick(true)+	unlock, err := w.lockRepo(r.key, r.dir)+	if err != nil {+		failed = err.Error()+		return+	}+	defer unlock()+	bare := filepath.Join(r.dir, ".git")+	start := time.Now()+	ctx, stop := w.bounded(r.dir)+	_, err = git(ctx, bare, "fetch", "--prune", "--no-write-fetch-head", "origin",+		"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")+	err = boundErr(ctx, err)+	stop()+	if err != nil {+		failed = err.Error()+		logErr("E%d: cannot fetch %s: %s", syscall.EHOSTUNREACH, r.key, failed)+		return+	}+	touch(filepath.Join(r.dir, ".fetched"))+	r.mu.Lock()+	old := r.tree.dir+	r.mu.Unlock()+	wt, err := w.checkout(r, nil, func(string) {})+	if err != nil {+		failed = err.Error()+		return+	}+	logInfo("fetched %s in %d ms", r.key, time.Since(start).Milliseconds())+	if wt == old {+		return+	}+	if err := w.show(r, wt); err != nil {+		failed = err.Error()+		return+	}+	w.dropTree(r, old)+	if err := w.trim(r.key); err != nil {+		failed = err.Error()+	}+}++// fetchWait bounds a public server's clone, fetch or checkout, so a forge+// that sends slowly cannot hold a clone's place for good.+const fetchWait = 10 * time.Minute++// bounded is the context for git fetching into the repo directory dir.+// On a public server it ends after fetchWait, or once dir outgrows the+// cache's limit: trim runs only after a fetch, and a stranger's forge may+// send without end.+func (w *web) bounded(dir string) (context.Context, func()) {+	if !w.public {+		return w.ctx, func() {}+	}+	ctx, cancel := context.WithCancelCause(w.ctx)+	ctx, cancelWait := context.WithTimeoutCause(ctx, fetchWait,+		&errno{syscall.ETIMEDOUT, fmt.Sprintf("the forge took over %d minutes, the most a public server waits", int(fetchWait.Minutes()))})+	done := make(chan struct{})+	w.wg.Go(func() {+		tick := time.NewTicker(2 * time.Second)+		defer tick.Stop()+		for {+			select {+			case <-done:+				return+			case <-tick.C:+				if n := diskUse(dir); n > w.limit {+					cancel(&errno{syscall.EFBIG, fmt.Sprintf("the repo grew past %s, the cache's limit, so the fetch stopped; --disk sets the limit", sizeText(w.limit))})+					return+				}+			}+		}+	})+	return ctx, func() {+		close(done)+		cancelWait()+		cancel(nil)+	}+}++// boundErr is why git failed in a context from bounded: the limit it hit,+// or err.+func boundErr(ctx context.Context, err error) error {+	if err != nil && ctx.Err() != nil {+		if cause := context.Cause(ctx); cause != nil {+			var e *errno+			if errors.As(cause, &e) {+				return cause+			}+		}+	}+	return err+}++// lockRepo marks the repo key busy and takes its lock.+func (w *web) lockRepo(key, dir string) (func(), error) {+	w.mu.Lock()+	w.busy[key] = true+	w.mu.Unlock()+	unlock, err := flockDir(dir, syscall.LOCK_EX)+	if err != nil {+		unlock = func() {}+	}+	return func() {+		unlock()+		w.mu.Lock()+		delete(w.busy, key)+		w.mu.Unlock()+	}, err+}++// flockDir takes the repo directory's lock, which keeps two servers from+// fetching one repo at once, and one from deleting a repo another fetches.+// how is LOCK_EX, or LOCK_EX|LOCK_NB to fail with EWOULDBLOCK when held.+func flockDir(dir string, how int) (func(), error) {+	f, err := os.OpenFile(filepath.Join(dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600)+	if err != nil {+		return nil, err+	}+	if err := flock(f, how); err != nil {+		f.Close()+		return nil, err+	}+	return func() { f.Close() }, nil+}