doc/online-repos.md

@@ -74,7 +74,9 @@ A visitor learns nothing of the machine or of what others read. `/` says only ho  No page has a stop or bug button, nothing opens a browser, and any Host header is answered, since nothing local is there to protect. -`--disk` is required, since any visitor can make www clone.+`--disk` is required, since any visitor can make www clone. A clone, fetch or checkout stops once the repo outgrows it or after 10 minutes, so a forge that sends without end cannot fill the disk or hold a clone's place.++A visitor can name any host, so www fetches only from hosts whose every address is on the open internet: never loopback, a private or link-local range, or carrier-grade NAT. git connects only to the addresses checked and follows no redirect, and listings check each address they connect to. A repo's `.git` is hidden, and a link out of a repo is refused without saying where it leads, whatever marker lies there.  The limits aim at a bot that makes www clone without end, not at a reader. A repo already in the cache opens at any time. A visit to an uncached path starts a read: a probe, a clone or a listing. 

git.go

@@ -83,6 +83,17 @@ func gitRun(ctx context.Context, dir string, stdin io.Reader, progress func(stri 	if remote { 		pre = append(pre, "-c", "protocol."+webScheme+".allow=always", "-c", "credential.helper=", "-c", "core.askPass=", 			"-c", "core.hooksPath="+os.DevNull)+		// git may fetch here, even for a page's blob, so a public server+		// checks where it would connect first.+		if publicNet.Load() {+			if host := gitHost(dir, args); host != "" {+				pin, err := reach(ctx, host)+				if err != nil {+					return nil, err+				}+				pre = append(pre, pin...)+			}+		} 	} 	args = append(append(pre, "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", "--no-pager", "--literal-pathspecs"), args...) 	cmd := exec.CommandContext(ctx, "git", args...)

links.go

@@ -29,6 +29,11 @@ func (f treeFS) Open(name string) (fs.File, error) { 	if f.t.shared && (isLockName(path.Base(name)) || f.t.dotted(name)) { 		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} 	}+	// A public tree hides the worktree's .git, which names the cache's+	// path.+	if f.t.public && f.t.inGit(name) {+		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}+	} 	file, err := f.open(name) 	if err != nil || !f.t.shared { 		return file, err@@ -44,7 +49,7 @@ func (f treeFS) Open(name string) (fs.File, error) { // lock, or on a share a dot entry. The listing and its version agree, so // a hidden file never reloads the page. func (t *tree) unlisted(dir, n string) bool {-	return isLockName(n) || t.shared && t.dotted(path.Join(dir, n))+	return isLockName(n) || t.shared && t.dotted(path.Join(dir, n)) || t.public && dir == "." && n == ".git" }  // dotted reports whether name, or where it resolves inside the tree,@@ -61,6 +66,21 @@ func (t *tree) dotted(name string) bool { 	return err == nil && hasDotName(filepath.ToSlash(rel)) } +// inGit reports whether name, or where it resolves inside the tree, is+// the worktree's .git or below it.+func (t *tree) inGit(name string) bool {+	isGit := func(n string) bool { return n == ".git" || strings.HasPrefix(n, ".git/") }+	if isGit(name) {+		return true+	}+	real, err := filepath.EvalSymlinks(filepath.Join(t.dir, filepath.FromSlash(name)))+	if err != nil || !within(real, t.realDir) {+		return false+	}+	rel, err := filepath.Rel(t.realDir, real)+	return err == nil && isGit(filepath.ToSlash(rel))+}+ // hasDotName reports whether a name on the way to name begins with a dot. func hasDotName(name string) bool { 	if name == "." {@@ -90,6 +110,13 @@ func (f treeFS) open(name string) (fs.File, error) { 		} 		return f.t.root.FS().Open(filepath.ToSlash(rel)) 	}+	// A stranger's repo never links out: a marker on the machine is+	// the owner's, not the repo's, to grant.+	if f.t.public {+		ref := &refusal{}+		ref.link, ref.target = f.t.outLink(name)+		return nil, ref+	} 	marker, ref := f.t.follow(name, real) 	if ref != nil { 		return nil, ref

reach.go

new file mode 100644@@ -0,0 +1,166 @@+package main++import (+	"context"+	"errors"+	"net"+	"net/http"+	"net/netip"+	"net/url"+	"path/filepath"+	"strings"+	"sync"+	"sync/atomic"+	"syscall"+	"time"+)++// A public server fetches whatever host a stranger names, so it must not+// reach the machine or the network it sits in: a probe of+// /127.0.0.1:6379/x, or a forge whose name leads to 10.0.0.1, would+// otherwise make it a stranger's way in. publicNet holds git and the+// listings to hosts whose every address lies on the open internet. git+// connects only to the addresses checked, so a name that answers again+// otherwise is not asked twice, and follows no redirect, which could+// lead anywhere.+var publicNet atomic.Bool++// reachable reports whether a public server may connect to a. Tests,+// whose forges listen on loopback, replace it.+var reachable = func(a netip.Addr) bool {+	a = a.Unmap()+	if !a.IsGlobalUnicast() || a.IsPrivate() {+		return false+	}+	for _, p := range unreachable {+		if p.Contains(a) {+			return false+		}+	}+	return true+}++// unreachable are the ranges that IsGlobalUnicast and IsPrivate pass but+// lead into a provider's or a site's own network.+var unreachable = []netip.Prefix{+	netip.MustParsePrefix("0.0.0.0/8"),+	netip.MustParsePrefix("100.64.0.0/10"), // carrier-grade NAT+	netip.MustParsePrefix("192.0.0.0/24"),+	netip.MustParsePrefix("198.18.0.0/15"),+	netip.MustParsePrefix("64:ff9b::/96"), // NAT64, to any IPv4 address+	netip.MustParsePrefix("64:ff9b:1::/48"),+	netip.MustParsePrefix("fec0::/10"), // site-local+}++// reachWait is how long a host's check holds, so pages that run git do+// not look the host up each time.+const reachWait = time.Minute++type reachAnswer struct {+	args []string+	err  error+	at   time.Time+}++var (+	reachMu    sync.Mutex+	reachCache = map[string]reachAnswer{}+)++// reach returns git's options that hold it to host's addresses, or why a+// public server may not reach host. host is as a URL names it, with a+// port or without.+func reach(ctx context.Context, host string) ([]string, error) {+	reachMu.Lock()+	a, ok := reachCache[host]+	reachMu.Unlock()+	if ok && time.Since(a.at) < reachWait {+		return a.args, a.err+	}+	args, err := lookupReach(ctx, host)+	if ctx.Err() != nil {+		return nil, err+	}+	reachMu.Lock()+	if len(reachCache) > 1000 {+		clear(reachCache)+	}+	reachCache[host] = reachAnswer{args, err, time.Now()}+	reachMu.Unlock()+	return args, err+}++func lookupReach(ctx context.Context, host string) ([]string, error) {+	name, port := host, ""+	if h, p, err := net.SplitHostPort(host); err == nil {+		name, port = h, p+	}+	name = strings.TrimSuffix(strings.TrimPrefix(name, "["), "]")+	if port == "" {+		port = "443"+		if webScheme == "http" {+			port = "80"+		}+	}+	args := []string{"-c", "http.followRedirects=false"}+	var addrs []netip.Addr+	if a, err := netip.ParseAddr(name); err == nil {+		addrs = []netip.Addr{a}+	} else {+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)+		defer cancel()+		if addrs, err = net.DefaultResolver.LookupNetIP(ctx, "ip", name); err != nil {+			return nil, &errno{syscall.EHOSTUNREACH, "cannot look up " + name + ": " + err.Error()}+		}+		var list []string+		for _, a := range addrs {+			if a = a.Unmap(); a.Is6() {+				list = append(list, "["+a.String()+"]")+			} else {+				list = append(list, a.String())+			}+		}+		args = append(args, "-c", "http.curloptResolve="+name+":"+port+":"+strings.Join(list, ","))+	}+	for _, a := range addrs {+		if !reachable(a) {+			return nil, &errno{syscall.EACCES, host + " leads to " + a.Unmap().String() + ", which is not on the open internet, so a public server does not fetch from it"}+		}+	}+	return args, nil+}++// gitHost is the host git reaches when it runs in dir with args: the+// host of a repo in the cache, or of the URL a probe names.+func gitHost(dir string, args []string) string {+	if dir != cacheRoot {+		rel, err := filepath.Rel(cacheRoot, dir)+		if err != nil {+			return ""+		}+		host, _, _ := strings.Cut(filepath.ToSlash(rel), "/")+		return host+	}+	for _, a := range args {+		if strings.HasPrefix(a, webScheme+"://") {+			if u, err := url.Parse(a); err == nil {+				return u.Host+			}+		}+	}+	return ""+}++// publicClient is webClient for a public server: it connects only to+// addresses on the open internet, wherever a name or a redirect leads,+// and through no proxy, which would hide where.+var publicClient = &http.Client{Timeout: 20 * time.Second, Transport: &http.Transport{+	DialContext: (&net.Dialer{Timeout: 10 * time.Second, Control: func(_, address string, _ syscall.RawConn) error {+		if ap, err := netip.ParseAddrPort(address); err != nil || !reachable(ap.Addr()) {+			return errors.New(address + " is not on the open internet")+		}+		return nil+	}}).DialContext,+	TLSHandshakeTimeout: 10 * time.Second,+	ForceAttemptHTTP2:   true,+}}

serve.go

@@ -434,6 +434,7 @@ func servePublic(c config) int { 		return rep.fail(errnoOf(err), "cannot serve online repos from "+root+": "+err.Error()) 	} 	w.public, w.srv.public = true, true+	publicNet.Store(true) 	w.slots = make(chan struct{}, cloneSlots) 	w.srv.live = newLive() 	httpd := &http.Server{Handler: w.srv.handler(), ReadHeaderTimeout: 10 * time.Second,

server.go

@@ -245,6 +245,7 @@ type tree struct { 	shared   bool        // served beyond loopback: only what other users could read 	secret   bool        // mounted under a token, and left off the index 	web      *webRepo    // the cached repo this tree is a worktree of, on the web address+	public   bool        // a stranger's repo on a public server: no link out, no path of the machine 	srv      *server     // the server that serves it, set when it joins 	linked   linkIndex   // the relative links of each doc, for ?backlinks 	at       *commit     // for a version, the commit it shows@@ -1211,7 +1212,7 @@ func (t *tree) serveDir(w http.ResponseWriter, r *http.Request, name string) { 			label += "/" 		} 		href := (&url.URL{Path: label}).String()-		if it.ref != nil && t.shared {+		if it.ref != nil && (t.shared || t.public) { 			fmt.Fprintf(&body, "<li class=\"refused\">%s <span class=\"own\">refused</span></li>\n", template.HTMLEscapeString(label)) 			continue 		}@@ -1369,10 +1370,11 @@ func searchURL(urlPath string) string { 	return (&url.URL{Path: urlPath[:strings.LastIndex(urlPath, "/")+1]}).String() } -// why is what a refusal page says: the reason in full, or on a shared tree-// only that the name is refused, since the reason names paths outside it.+// why is what a refusal page says: the reason in full, or on a shared or+// public tree only that the name is refused, since the reason names paths+// outside it. func (t *tree) why(ref *refusal) string {-	if t.shared {+	if t.shared || t.public { 		return "Refused: " + ref.link 	} 	return ref.Error()

version.go

@@ -106,7 +106,7 @@ func (t *tree) atVersion(ctx context.Context, seg string) (*tree, string, error) 	if v, ok := t.versions.Load(best); ok { 		return v.(*tree), best, nil 	}-	v := &tree{dir: t.dir, realDir: t.realDir, prefix: versionPrefix(t.prefix, best), label: t.label, wd: -1, at: &c, atLabel: stars.label(&c), base: t}+	v := &tree{dir: t.dir, realDir: t.realDir, prefix: versionPrefix(t.prefix, best), label: t.label, wd: -1, at: &c, atLabel: stars.label(&c), base: t, public: t.public} 	v.fsys = commitFS{dir: t.dir, rev: c.hash, when: c.date} 	got, _ := t.versions.LoadOrStore(best, v) 	return got.(*tree), best, nil

web.go

@@ -410,7 +410,7 @@ func (w *web) show(r *webRepo, wt string) error { 		return err 	} 	t := newTree(root, wt, "/"+r.key+"/", r.key)-	t.web = r+	t.web, t.public = r, w.public 	r.mu.Lock() 	old := r.tree 	r.tree = t@@ -542,6 +542,12 @@ func (w *web) end(j *webJob, err error) { // run finds the repo the path lies in and clones it, or lists the path // when it names no repo. func (w *web) run(j *webJob, key string, segs []string) {+	if publicNet.Load() {+		if _, err := reach(w.ctx, segs[0]); err != nil {+			w.end(j, err)+			return+		}+	} 	repoKey, cloneURL, probed := w.probe(j, segs) 	if repoKey == "" { 		w.list(j, key, probed)@@ -827,7 +833,11 @@ func (w *web) clone(j *webJob, key, cloneURL string) error { 		os.RemoveAll(tmp) 		w.step(j, "git clone --bare --filter=blob:none "+cloneURL+" "+w.where(bare)) 		start := time.Now()-		if _, err := gitProgress(w.ctx, dir, w.progress(j), "clone", "--bare", "--filter=blob:none", "--progress", "--", cloneURL, tmp); err != nil {+		ctx, stop := w.bounded(dir)+		_, err := gitProgress(ctx, dir, w.progress(j), "clone", "--bare", "--filter=blob:none", "--progress", "--", cloneURL, tmp)+		err = boundErr(ctx, err)+		stop()+		if err != nil { 			os.RemoveAll(tmp) 			return err 		}@@ -864,7 +874,11 @@ func (w *web) checkout(r *webRepo, progress func(string), step func(string)) (st 	if !exists(wt) { 		step("git worktree add --detach " + w.where(wt) + " " + head[:12] + ", which fetches its files") 		start := time.Now()-		if _, err := gitProgress(w.ctx, bare, progress, "worktree", "add", "--detach", "--", wt, head); err != nil {+		ctx, stop := w.bounded(r.dir)+		_, err := gitProgress(ctx, bare, progress, "worktree", "add", "--detach", "--", wt, head)+		err = boundErr(ctx, err)+		stop()+		if err != nil { 			os.RemoveAll(wt) 			git(w.ctx, bare, "worktree", "prune") 			return "", err@@ -901,8 +915,12 @@ func (w *web) refresh(r *webRepo) { 	defer unlock() 	bare := filepath.Join(r.dir, ".git") 	start := time.Now()-	if _, err := git(w.ctx, bare, "fetch", "--prune", "--no-write-fetch-head", "origin",-		"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"); err != nil {+	ctx, stop := w.bounded(r.dir)+	_, err = git(ctx, bare, "fetch", "--prune", "--no-write-fetch-head", "origin",+		"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")+	err = boundErr(ctx, err)+	stop()+	if err != nil { 		failed = err.Error() 		logErr("E%d: cannot fetch %s: %s", syscall.EHOSTUNREACH, r.key, failed) 		return@@ -930,6 +948,58 @@ func (w *web) refresh(r *webRepo) { 	} } +// fetchWait bounds a public server's clone, fetch or checkout, so a forge+// that sends slowly cannot hold a clone's place for good.+const fetchWait = 10 * time.Minute++// bounded is the context for git fetching into the repo directory dir.+// On a public server it ends after fetchWait, or once dir outgrows the+// cache's limit: trim runs only after a fetch, and a stranger's forge may+// send without end.+func (w *web) bounded(dir string) (context.Context, func()) {+	if !w.public {+		return w.ctx, func() {}+	}+	ctx, cancel := context.WithCancelCause(w.ctx)+	ctx, cancelWait := context.WithTimeoutCause(ctx, fetchWait,+		&errno{syscall.ETIMEDOUT, fmt.Sprintf("the forge took over %d minutes, the most a public server waits", int(fetchWait.Minutes()))})+	done := make(chan struct{})+	w.wg.Go(func() {+		tick := time.NewTicker(2 * time.Second)+		defer tick.Stop()+		for {+			select {+			case <-done:+				return+			case <-tick.C:+				if n := diskUse(dir); n > w.limit {+					cancel(&errno{syscall.EFBIG, fmt.Sprintf("the repo grew past %s, the cache's limit, so the fetch stopped; --disk sets the limit", sizeText(w.limit))})+					return+				}+			}+		}+	})+	return ctx, func() {+		close(done)+		cancelWait()+		cancel(nil)+	}+}++// boundErr is why git failed in a context from bounded: the limit it hit,+// or err.+func boundErr(ctx context.Context, err error) error {+	if err != nil && ctx.Err() != nil {+		if cause := context.Cause(ctx); cause != nil {+			var e *errno+			if errors.As(cause, &e) {+				return cause+			}+		}+	}+	return err+}+ // lockRepo marks the repo key busy and takes its lock. func (w *web) lockRepo(key, dir string) (func(), error) { 	w.mu.Lock()@@ -1322,7 +1392,11 @@ func (w *web) ask(src listSource) ([]listed, error) { 			return nil, err 		} 		req.Header.Set("User-Agent", progName)-		resp, err := webClient.Do(req)+		client := webClient+		if w.public {+			client = publicClient+		}+		resp, err := client.Do(req) 		if err != nil { 			return nil, err 		}

web_test.go

@@ -4,6 +4,7 @@ import ( 	"context" 	"fmt" 	"io"+	"net" 	"net/http" 	"net/http/cgi" 	"net/http/httptest"@@ -12,6 +13,7 @@ import ( 	"os" 	"os/exec" 	"path/filepath"+	"slices" 	"strings" 	"sync/atomic" 	"syscall"@@ -353,7 +355,7 @@ func TestWebPublic(t *testing.T) { 	if body := get("/").Body.String(); strings.Contains(body, "acct/r") || strings.Contains(body, "The cache") { 		t.Errorf("the root shows the cache:\n%s", body) 	}-	if rec := read(t, w, "/"+f.host+"/acct"); rec.Code != http.StatusNotFound || strings.Contains(rec.Body.String(), "acct/r") {+	if rec := read(t, w, "/"+f.host+"/acct"); rec.Code != http.StatusNotFound || strings.Contains(rec.Body.String(), "acct/r/\"") { 		t.Errorf("a listing falls back to the cache: %d\n%s", rec.Code, rec.Body) 	} 	if rec := get("/" + f.host + "/acct/r/README.md"); rec.Code != http.StatusOK {@@ -628,3 +630,74 @@ func TestWebCloneWaitsForASlot(t *testing.T) { 		t.Errorf("clone after a slot frees: %d\n%s", rec.Code, rec.Body) 	} }++// A public server reaches no address of the machine or its network, by+// an address or by a name, and holds git to the addresses it checked.+func TestWebPublicReach(t *testing.T) {+	f := newForge(t, nil)+	f.repo(t, "acct/r", map[string]string{"README.md": "x\n"})+	w := testWeb(t, defaultDisk)+	w.public, w.srv.public = true, true+	publicNet.Store(true)+	t.Cleanup(func() { publicNet.Store(false) })+	clear(reachCache)+	_, port, _ := net.SplitHostPort(f.host)+	for _, p := range []string{"/" + f.host + "/acct/r/", "/localhost:" + port + "/acct/r/", "/169.254.169.254/latest/meta-data", "/[::1]:" + port + "/acct/r/"} {+		if rec := read(t, w, p); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "not on the open internet") {+			t.Errorf("%s: %d\n%s", p, rec.Code, rec.Body)+		}+	}+	if _, err := publicClient.Get("http://" + f.host + "/"); err == nil || !strings.Contains(err.Error(), "not on the open internet") {+		t.Errorf("listing client reached loopback: %v", err)+	}+	// Where loopback counts as the open internet, git reaches the forge+	// by its name at the address checked.+	old := reachable+	reachable = func(netip.Addr) bool { return true }+	t.Cleanup(func() { reachable = old })+	clear(reachCache)+	w2 := testWeb(t, defaultDisk)+	w2.public, w2.srv.public = true, true+	depths["localhost:"+port] = 3+	t.Cleanup(func() { delete(depths, "localhost:"+port) })+	if rec := read(t, w2, "/localhost:"+port+"/acct/r/README.md"); rec.Code != http.StatusOK {+		t.Errorf("a forge by name: %d\n%s", rec.Code, rec.Body)+	}+	if args, _ := reach(context.Background(), "localhost:"+port); !slices.Contains(args, "http.followRedirects=false") ||+		!slices.ContainsFunc(args, func(a string) bool { return strings.HasPrefix(a, "http.curloptResolve=localhost:"+port+":") }) {+		t.Errorf("git's options: %q", args)+	}+}++// A public repo shows nothing of the machine: not its .git, which names+// the cache, nor where a link out leads, which it never follows.+func TestWebPublicHidesTheMachine(t *testing.T) {+	f := newForge(t, nil)+	f.repo(t, "acct/r", map[string]string{"README.md": "x\n"})+	work := f.work["acct/r"]+	for name, to := range map[string]string{"up": "../../../../../..", "etc": "/etc", "g": ".git", "gc": ".git/../.git"} {+		if err := os.Symlink(to, filepath.Join(work, name)); err != nil {+			t.Fatal(err)+		}+	}+	commitAll(t, work, "links")+	if out, err := exec.Command("git", "-C", work, "push", "-q", "origin", "main").CombinedOutput(); err != nil {+		t.Fatalf("git push: %v\n%s", err, out)+	}+	w := testWeb(t, defaultDisk)+	w.public, w.srv.public = true, true+	// A marker above the cache would open a link out on a reader's+	// server, but not on a public one.+	touch(filepath.Join(filepath.Dir(w.root), followMarker))+	read(t, w, "/"+f.host+"/acct/r/")+	for _, p := range []string{"", ".git", "g", "gc", "up/", "etc/", "etc/passwd", "up/?log"} {+		rec := get(w.srv.handler(), "/"+f.host+"/acct/r/"+p, "Host", "visitor.example.org")+		body := rec.Body.String()+		if strings.Contains(body, filepath.Dir(w.root)) || strings.Contains(body, "gitdir") || strings.Contains(body, "root:") {+			t.Errorf("%s shows the machine: %d\n%s", p, rec.Code, body)+		}+		if p == "" && strings.Contains(body, ">.git<") {+			t.Errorf("the listing shows .git:\n%s", body)+		}+	}+}