fastwalk.go

@@ -77,18 +77,23 @@ func DefaultNumWorkers() int { 	if numCPU < 4 { 		return 4 	}+	// User manually set GOMAXPROCS - respect it.+	if numCPU != runtime.NumCPU() {+		return min(numCPU, 32)+	} 	// Darwin IO performance on APFS can slow with increased parallelism.-	// Depending on CPU, stat(2) performance is best around 4-10 workers-	// and file IO is best around 4 workers. More workers only benefit CPU-	// intensive tasks.-	//-	// NB(Charlie): As of macOS 15, the parallel performance of readdir_r(3)-	// and stat(2) calls has improved and is now generally the number of-	// performance cores (on ARM Macs).+	// For Intel CPUs (and maybe older arm64 CPUs) performance is best+	// around 4-10 workers and file IO is best around 4 workers. More workers+	// only benefit CPU intensive tasks. 	//-	// TODO: Consider using the value of sysctl("hw.perflevel0.physicalcpu").-	// TODO: Find someone with a Mac Studio to test higher core counts.+	// As of macOS 15 (on ARM Macs), the parallel performance of readdir_r(3)+	// and stat(2) calls has improved and the ideal number of workers is now+	// generally the number of performance cores. 	if runtime.GOOS == "darwin" {+		if n := darwinNumPerfCores(); n > 0 {+			return n+		}+		// This is primarily for Intel CPUs. 		switch { 		case numCPU <= 8: 			return 4@@ -98,10 +103,7 @@ func DefaultNumWorkers() int { 			return 10 		} 	}-	if numCPU > 32 {-		return 32-	}-	return numCPU+	return min(numCPU, 32) }  // DefaultToSlash returns true if this is a Go program compiled for Windows

fastwalk_test.go

@@ -10,11 +10,13 @@ import ( 	"io/fs" 	"math" 	"os"+	"os/exec" 	"os/user" 	"path/filepath" 	"reflect" 	"regexp" 	"runtime"+	"slices" 	"sort" 	"strconv" 	"strings"@@ -1343,6 +1345,83 @@ func TestSkipAll(t *testing.T) { 	} } +func TestDefaultNumWorkers(t *testing.T) {+	if os.Getenv("FASTWALK_TEST_COOKIE") == "123456" {+		fmt.Printf("## DEFAULT_NUM_WORKERS: %d\n", fastwalk.DefaultNumWorkers())+		return+	}++	exe, err := os.Executable()+	if err != nil {+		t.Fatal(err)+	}++	re := regexp.MustCompile(`## DEFAULT_NUM_WORKERS:\s+(\d+)`)+	parseOut := func(t *testing.T, out []byte) int {+		a := re.FindSubmatch(out)+		if len(a) != 2 {+			t.Fatal(`Sub-test output does not contain: "## DEFAULT_NUM_WORKERS"`)+		}+		n, err := strconv.Atoi(string(a[1]))+		if err != nil {+			t.Fatal(err)+		}+		return n+	}++	runTest := func(t *testing.T, maxProcs, want int) {+		cmd := exec.Command(exe, "-test.run", "^TestDefaultNumWorkers$")+		cmd.Env = cmd.Environ()+		cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool {+			return strings.HasPrefix(s, "GOMAXPROCS=") || strings.HasPrefix(s, "GOFLAGS=")+		})+		cmd.Env = append(cmd.Env, "FASTWALK_TEST_COOKIE=123456")+		if maxProcs > 0 {+			cmd.Env = append(cmd.Env, fmt.Sprintf("GOMAXPROCS=%d", maxProcs))+		}+		out, err := cmd.CombinedOutput()+		if err != nil {+			t.Fatalf("test failed: %s", bytes.TrimSpace(out))+		}+		got := parseOut(t, out)+		if got != want {+			t.Fatalf("got: %d want: %d", got, want)+		}+	}++	t.Run("GOMAXPROCS", func(t *testing.T) {+		for _, procs := range []int{2, 4, 8, 20, 64} {+			t.Run(strconv.Itoa(procs), func(t *testing.T) {+				want := max(4, min(procs, 32))+				runTest(t, procs, want)+			})+		}+	})++	t.Run("Darwin_ARM64", func(t *testing.T) {+		if runtime.GOOS != "darwin" && runtime.GOARCH != "arm64" {+			t.Skip("test only supported on darwin/arm64")+		}+		// Not all platforms have syscall.SysctlUint32 so shell out to+		// "sysctl". This is ugly but saves us from having to duplicate+		// a large portion of this code.+		out, err := exec.Command("sysctl", "hw.perflevel0.physicalcpu").CombinedOutput()+		if err != nil {+			t.Fatal(err)+		}+		_, val, ok := strings.Cut(strings.TrimSpace(string(out)), " ")+		if !ok {+			t.Fatalf("Invalid sysctl output: %q", out)+		}+		want, err := strconv.Atoi(val)+		if err != nil {+			t.Fatal(err)+		}+		want = max(want, 4) // 4 is the minimum+		runTest(t, -1, int(want))+	})+}+ func BenchmarkSortModeString(b *testing.B) { 	var s string 	for i := 0; i < b.N; i++ {

numcpu_darwin_arm64.go

new file mode 100644@@ -0,0 +1,25 @@+//go:build darwin && arm64++package fastwalk++import "syscall"++// darwinNumPerfCores returns the number of performance cores currently+// available on macOS/arm64.+func darwinNumPerfCores() int {+	// "hw.physicalcpu" is the number of physical processors available in+	// the current power management mode.+	//+	// https://github.com/apple-oss-distributions/xnu/blob/43a90889846e00bfb5cf1d255cdc0a701a1e05a4/bsd/sys/sysctl.h#L1244+	//+	// NB: We do not cache this value since it could change with the power+	// management mode.+	//+	// TODO: Find someone with a Mac Studio or MacPro to see if this still+	// holds true when the CPU is basically two "fused" ones.+	n, err := syscall.SysctlUint32("hw.perflevel0.physicalcpu")+	if err != nil {+		return -1+	}+	return int(n)+}

numcpu_darwin_arm64_test.go

new file mode 100644@@ -0,0 +1,11 @@+package fastwalk++import "testing"++func TestDarwinNumPerfCores(t *testing.T) {+	n := darwinNumPerfCores()+	// Test that n is reasonable+	if !(0 < n && n < 4096) {+		t.Fatalf("expected a value between 0..4096 got: %d", n)+	}+}

numcpu_portable.go

new file mode 100644@@ -0,0 +1,7 @@+//go:build !(darwin && arm64)++package fastwalk++func darwinNumPerfCores() int {+	return -1+}