current commit
261101.2 · web.go splits by purpose
package main

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

// probe asks every prefix of the path at once whether a repo answers
// there, each with and without .git, and takes the longest that does. It
// returns the repo's key and URL, or "" and whether it probed at all.
func (w *web) probe(j *webJob, segs []string) (key, cloneURL string, probed bool) {
	lo, hi := 2, min(len(segs), probeMax+1)
	if d, ok := depths[segs[0]]; ok {
		// The repo is the first d segments, so a probe would tell
		// nothing the clone does not, and costs a round trip.
		if len(segs) >= d {
			key = strings.Join(segs[:d], "/")
			w.step(j, segs[0]+" keeps every repo at owner/name, so "+webScheme+"://"+key+" needs no probe")
			return key, webScheme + "://" + key, true
		}
		lo, hi = d, min(d, len(segs))
	} else if d, ok := minDepths[segs[0]]; ok {
		lo = d
	}
	if lo > hi {
		return "", "", false
	}
	type answer struct {
		n   int
		url string
	}
	var urls []string
	for n := hi; n >= lo; n-- {
		u := webScheme + "://" + strings.Join(segs[:n], "/")
		urls = append(urls, u)
	}
	w.step(j, "git ls-remote, at once, on "+strings.Join(urls, " "))
	start := time.Now()
	ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second)
	defer cancel()
	ch := make(chan answer)
	for n := hi; n >= lo; n-- {
		for _, u := range []string{webScheme + "://" + strings.Join(segs[:n], "/"), webScheme + "://" + strings.Join(segs[:n], "/") + ".git"} {
			go func() {
				if _, err := git(ctx, w.root, "ls-remote", u, "HEAD"); err == nil {
					ch <- answer{n, u}
				} else {
					ch <- answer{0, ""}
				}
			}()
		}
	}
	var best answer
	for range 2 * (hi - lo + 1) {
		if a := <-ch; a.n > best.n || a.n == best.n && a.n > 0 && len(a.url) < len(best.url) {
			best = a
		}
	}
	ms := time.Since(start).Milliseconds()
	if best.n == 0 {
		w.step(j, fmt.Sprintf("no repo answered, in %d ms", ms))
		return "", "", true
	}
	w.step(j, fmt.Sprintf("%s answered, in %d ms", best.url, ms))
	return strings.Join(segs[:best.n], "/"), best.url, true
}

// clone fetches the repo key from cloneURL into the cache and serves it.
func (w *web) clone(j *webJob, key, cloneURL string) error {
	w.mu.Lock()
	r := w.repos[key]
	w.mu.Unlock()
	if r != nil {
		return nil
	}
	dir := filepath.Join(w.root, filepath.FromSlash(key))
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return err
	}
	unlock, err := w.lockRepo(key, dir)
	if err != nil {
		return err
	}
	defer unlock()
	// Another visit or server may have cloned it meanwhile.
	if r, err := w.load(key, true); r != nil || err != nil {
		return err
	}
	bare := filepath.Join(dir, ".git")
	if !exists(filepath.Join(bare, "HEAD")) {
		if w.slots != nil {
			select {
			case w.slots <- struct{}{}:
			default:
				w.step(j, fmt.Sprintf("waiting: %d clones run at once", cap(w.slots)))
				select {
				case w.slots <- struct{}{}:
				case <-w.ctx.Done():
					return w.ctx.Err()
				}
			}
			// The slot covers the worktree's blobs too, the larger fetch.
			defer func() { <-w.slots }()
		}
		tmp := bare + ".tmp"
		os.RemoveAll(tmp)
		w.step(j, "git clone --bare --filter=blob:none "+cloneURL+" "+w.where(bare))
		start := time.Now()
		ctx, stop := w.bounded(dir)
		_, err := gitProgress(ctx, dir, w.progress(j), "clone", "--bare", "--filter=blob:none", "--progress", "--", cloneURL, tmp)
		err = boundErr(ctx, err)
		stop()
		if err != nil {
			os.RemoveAll(tmp)
			return err
		}
		if err := os.Rename(tmp, bare); err != nil {
			return err
		}
		w.step(j, fmt.Sprintf("cloned in %s", seconds(time.Since(start))))
	}
	r = &webRepo{key: key, dir: dir}
	touch(filepath.Join(dir, ".fetched"))
	touch(filepath.Join(dir, ".used"))
	if _, err := w.checkout(r, w.progress(j), func(s string) { w.step(j, s) }); err != nil {
		return err
	}
	// The cache must hold the new repo before a page shows it.
	if err := w.trim(key); err != nil {
		return err
	}
	_, err = w.load(key, true)
	return err
}

// checkout builds the worktree of the commit HEAD names, unless it is
// there, and points .tree at it.
func (w *web) checkout(r *webRepo, progress func(string), step func(string)) (string, error) {
	bare := filepath.Join(r.dir, ".git")
	out, err := git(w.ctx, bare, "rev-parse", "--verify", "HEAD^{commit}")
	if err != nil {
		return "", err
	}
	head := strings.TrimSpace(string(out))
	name := ".wt-" + head[:12]
	wt := filepath.Join(r.dir, name)
	if !exists(wt) {
		step("git worktree add --detach " + w.where(wt) + " " + head[:12] + ", which fetches its files")
		start := time.Now()
		ctx, stop := w.bounded(r.dir)
		_, err := gitProgress(ctx, bare, progress, "worktree", "add", "--detach", "--", wt, head)
		err = boundErr(ctx, err)
		stop()
		if err != nil {
			os.RemoveAll(wt)
			git(w.ctx, bare, "worktree", "prune")
			return "", err
		}
		step(fmt.Sprintf("checked out in %s", seconds(time.Since(start))))
	}
	link := filepath.Join(r.dir, ".tree")
	os.Remove(link + ".new")
	if err := os.Symlink(name, link+".new"); err != nil {
		return "", err
	}
	return wt, os.Rename(link+".new", link)
}

// refresh fetches r again, and serves the new commit when HEAD moved. A
// failed fetch leaves the commit served, and its pages say so.
func (w *web) refresh(r *webRepo) {
	failed := ""
	defer func() {
		r.mu.Lock()
		r.fetching, r.failed = false, failed
		if failed == "" {
			r.fetched = time.Now()
		}
		r.mu.Unlock()
		w.kick(true)
	}()
	w.kick(true)
	unlock, err := w.lockRepo(r.key, r.dir)
	if err != nil {
		failed = err.Error()
		return
	}
	defer unlock()
	bare := filepath.Join(r.dir, ".git")
	start := time.Now()
	ctx, stop := w.bounded(r.dir)
	_, err = git(ctx, bare, "fetch", "--prune", "--no-write-fetch-head", "origin",
		"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")
	err = boundErr(ctx, err)
	stop()
	if err != nil {
		failed = err.Error()
		logErr("E%d: cannot fetch %s: %s", syscall.EHOSTUNREACH, r.key, failed)
		return
	}
	touch(filepath.Join(r.dir, ".fetched"))
	r.mu.Lock()
	old := r.tree.dir
	r.mu.Unlock()
	wt, err := w.checkout(r, nil, func(string) {})
	if err != nil {
		failed = err.Error()
		return
	}
	logInfo("fetched %s in %d ms", r.key, time.Since(start).Milliseconds())
	if wt == old {
		return
	}
	if err := w.show(r, wt); err != nil {
		failed = err.Error()
		return
	}
	w.dropTree(r, old)
	if err := w.trim(r.key); err != nil {
		failed = err.Error()
	}
}

// fetchWait bounds a public server's clone, fetch or checkout, so a forge
// that sends slowly cannot hold a clone's place for good.
const fetchWait = 10 * time.Minute

// bounded is the context for git fetching into the repo directory dir.
// On a public server it ends after fetchWait, or once dir outgrows the
// cache's limit: trim runs only after a fetch, and a stranger's forge may
// send without end.
func (w *web) bounded(dir string) (context.Context, func()) {
	if !w.public {
		return w.ctx, func() {}
	}
	ctx, cancel := context.WithCancelCause(w.ctx)
	ctx, cancelWait := context.WithTimeoutCause(ctx, fetchWait,
		&errno{syscall.ETIMEDOUT, fmt.Sprintf("the forge took over %d minutes, the most a public server waits", int(fetchWait.Minutes()))})
	done := make(chan struct{})
	w.wg.Go(func() {
		tick := time.NewTicker(2 * time.Second)
		defer tick.Stop()
		for {
			select {
			case <-done:
				return
			case <-tick.C:
				if n := diskUse(dir); n > w.limit {
					cancel(&errno{syscall.EFBIG, fmt.Sprintf("the repo grew past %s, the cache's limit, so the fetch stopped; --disk sets the limit", sizeText(w.limit))})
					return
				}
			}
		}
	})
	return ctx, func() {
		close(done)
		cancelWait()
		cancel(nil)
	}
}

// boundErr is why git failed in a context from bounded: the limit it hit,
// or err.
func boundErr(ctx context.Context, err error) error {
	if err != nil && ctx.Err() != nil {
		if cause := context.Cause(ctx); cause != nil {
			var e *errno
			if errors.As(cause, &e) {
				return cause
			}
		}
	}
	return err
}

// lockRepo marks the repo key busy and takes its lock.
func (w *web) lockRepo(key, dir string) (func(), error) {
	w.mu.Lock()
	w.busy[key] = true
	w.mu.Unlock()
	unlock, err := flockDir(dir, syscall.LOCK_EX)
	if err != nil {
		unlock = func() {}
	}
	return func() {
		unlock()
		w.mu.Lock()
		delete(w.busy, key)
		w.mu.Unlock()
	}, err
}

// flockDir takes the repo directory's lock, which keeps two servers from
// fetching one repo at once, and one from deleting a repo another fetches.
// how is LOCK_EX, or LOCK_EX|LOCK_NB to fail with EWOULDBLOCK when held.
func flockDir(dir string, how int) (func(), error) {
	f, err := os.OpenFile(filepath.Join(dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600)
	if err != nil {
		return nil, err
	}
	if err := flock(f, how); err != nil {
		f.Close()
		return nil, err
	}
	return func() { f.Close() }, nil
}