Changes to weblist.go. All files

weblist.go

new file mode 100644@@ -0,0 +1,212 @@+package main++import (+	"bufio"+	"encoding/json"+	"errors"+	"io"+	"net/http"+	"net/url"+	"regexp"+	"slices"+	"strconv"+	"strings"+	"time"+)++// listing is the repos a host or account holds, and what told.+type listing struct {+	at     time.Time+	source string   // what answered, eg "the GitHub API"+	asked  string   // the URL asked+	failed []string // the sources that failed before it, and why+	repos  []listed+}++type listed struct{ key, desc string }++func (l *listing) fresh() time.Duration {+	if len(l.repos) == 0 {+		return missFresh+	}+	return listFresh+}++// list lists the path key, by the first source that answers: the forge's+// API, a web frontend's index, then the cache.+func (w *web) list(j *webJob, key string, probed bool) {+	segs := strings.Split(key, "/")+	l := &listing{at: time.Now()}+	for _, src := range listSources(segs) {+		w.step(j, "GET "+src.url+", "+src.name)+		repos, err := w.ask(src)+		if err == nil && len(repos) > 0 {+			l.source, l.asked, l.repos = src.name, src.url, repos+			break+		}+		if err == nil {+			err = errors.New("no repos")+		}+		l.failed = append(l.failed, src.name+": "+err.Error())+		w.step(j, src.name+": "+err.Error())+	}+	// A stranger must not learn what others read, so a public server+	// never lists the cache.+	if l.source == "" && !w.public {+		l.source = "the cache"+		for _, c := range w.onDisk() {+			if strings.HasPrefix(c.key, key+"/") {+				l.repos = append(l.repos, listed{key: c.key})+			}+		}+		slices.SortFunc(l.repos, func(a, b listed) int { return strings.Compare(a.key, b.key) })+	}+	if len(l.repos) == 0 && probed && len(l.failed) == 0 {+		l.failed = append(l.failed, "no source lists "+key)+	}+	logInfo("listed %d repos at %s by %s", len(l.repos), key, l.source)+	w.mu.Lock()+	w.lists[key] = l+	w.mu.Unlock()+	w.end(j, nil)+}++// listSource is one place that may list a path's repos.+type listSource struct {+	name, url string+	parse     func(body []byte, next *string) ([]listed, error)+}++var (+	cgitRepo   = regexp.MustCompile(`<td class='(?:top|sub)level-repo'><a (?:title='[^']*' )?href='([^']+)'`)+	stagitRepo = regexp.MustCompile(`<td><a href="([^"/]+)/log\.html">`)+)++// listSources are the sources for the path segs, in the order to ask them.+func listSources(segs []string) []listSource {+	host, rest := segs[0], strings.Join(segs[1:], "/")+	var out []listSource+	jsonList := func(field string) func([]byte, *string) ([]listed, error) {+		return func(b []byte, _ *string) ([]listed, error) {+			var v []map[string]any+			if err := json.Unmarshal(b, &v); err != nil {+				return nil, errors.New("not a list of repos")+			}+			var out []listed+			for _, m := range v {+				name, _ := m[field].(string)+				desc, _ := m["description"].(string)+				if name != "" {+					out = append(out, listed{key: host + "/" + name, desc: desc})+				}+			}+			return out, nil+		}+	}+	switch {+	case host == "github.com" && len(segs) == 2:+		out = append(out, listSource{"the GitHub API", "https://api.github.com/users/" + rest + "/repos?per_page=100", jsonList("full_name")})+	case host == "gitlab.com" && len(segs) >= 2:+		if len(segs) == 2 {+			out = append(out, listSource{"the GitLab API", "https://gitlab.com/api/v4/users/" + rest + "/projects?per_page=100", jsonList("path_with_namespace")})+		}+		out = append(out, listSource{"the GitLab API", "https://gitlab.com/api/v4/groups/" + url.PathEscape(rest) + "/projects?per_page=100&include_subgroups=true", jsonList("path_with_namespace")})+	case host == "bitbucket.org" && len(segs) == 2:+		out = append(out, listSource{"the Bitbucket API", "https://api.bitbucket.org/2.0/repositories/" + rest + "?pagelen=100",+			func(b []byte, next *string) ([]listed, error) {+				var v struct {+					Values []struct{ Full_name, Description string }+					Next   string+				}+				if err := json.Unmarshal(b, &v); err != nil {+					return nil, errors.New("not a list of repos")+				}+				*next = v.Next+				var out []listed+				for _, r := range v.Values {+					out = append(out, listed{key: host + "/" + r.Full_name, desc: r.Description})+				}+				return out, nil+			}})+	case len(segs) == 2 && !slices.Contains([]string{"github.com", "gitlab.com", "bitbucket.org"}, host):+		out = append(out, listSource{"the Forgejo API", webScheme + "://" + host + "/api/v1/users/" + rest + "/repos?limit=50", jsonList("full_name")})+	}+	if _, forge := depths[host]; forge || host == "gitlab.com" {+		return out+	}+	base := webScheme + "://" + strings.Join(segs, "/") + "/"+	out = append(out, listSource{"gitweb's project index", base + "?a=project_index", func(b []byte, _ *string) ([]listed, error) {+		var out []listed+		sc := bufio.NewScanner(strings.NewReader(string(b)))+		for sc.Scan() {+			f := strings.Fields(sc.Text())+			if len(f) == 0 || strings.ContainsAny(f[0], "<>\"") {+				return nil, errors.New("not a project index")+			}+			name, err := url.QueryUnescape(f[0])+			if err != nil {+				return nil, errors.New("not a project index")+			}+			out = append(out, listed{key: strings.Join(segs, "/") + "/" + strings.TrimSuffix(name, ".git")})+		}+		return out, nil+	}})+	out = append(out, listSource{"the frontend's front page", base, func(b []byte, _ *string) ([]listed, error) {+		var out []listed+		for _, m := range cgitRepo.FindAllSubmatch(b, -1) {+			p := strings.Trim(string(m[1]), "/")+			out = append(out, listed{key: host + "/" + strings.TrimSuffix(p, ".git")})+		}+		for _, m := range stagitRepo.FindAllSubmatch(b, -1) {+			out = append(out, listed{key: strings.Join(segs, "/") + "/" + string(m[1])})+		}+		return out, nil+	}})+	return out+}++var webClient = &http.Client{Timeout: 20 * time.Second}++// ask reads src, following the forge's pages up to 10.+func (w *web) ask(src listSource) ([]listed, error) {+	var out []listed+	u := src.url+	for page := 1; page <= 10 && u != ""; page++ {+		req, err := http.NewRequestWithContext(w.ctx, "GET", u, nil)+		if err != nil {+			return nil, err+		}+		req.Header.Set("User-Agent", progName)+		client := webClient+		if w.public {+			client = publicClient+		}+		resp, err := client.Do(req)+		if err != nil {+			return nil, err+		}+		b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))+		resp.Body.Close()+		if err != nil {+			return nil, err+		}+		if resp.StatusCode != http.StatusOK {+			return nil, errors.New(resp.Status)+		}+		next := ""+		got, err := src.parse(b, &next)+		if err != nil {+			return nil, err+		}+		out = append(out, got...)+		switch {+		case next != "":+			u = next+		case strings.Contains(src.url, "api") && len(got) >= 50 && !strings.Contains(src.url, "bitbucket"):+			u = src.url + "&page=" + strconv.Itoa(page+1)+		default:+			u = ""+		}+	}+	return out, nil+}