mirrors/Mole
1
0
Fork 0
mirror of https://github.com/tw93/Mole.git synced 2026-02-15 23:11:11 +00:00
Code Activity

Create separate limiter for du subprocesses

Browse Source 
Introduce duSem to cap concurrent du subprocess spawns while keeping
the existing sem to limit overall scan fan-out (dir recursion/task concurrency).
This prevents du from monopolizing scan concurrency and reduces resource
spikes during folded-dir sizing.
This commit is contained in:
Jack Phallen
2026-01-11 10:56:18 -05:00
parent cf4690191e
commit b8ab9511fd
1 changed files with 15 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
cmd/analyze/scanner.go
View File

@@ -50,6 +50,7 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
numWorkers = 1 numWorkers = 1
} }
sem := make(chan struct{}, numWorkers) sem := make(chan struct{}, numWorkers)
duSem := make(chan struct{}, min(4, runtime.NumCPU()))
var wg sync.WaitGroup var wg sync.WaitGroup
// Collect results via channels. // Collect results via channels.
@@ -138,7 +139,7 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
} else if cached, err := loadCacheFromDisk(path); err == nil { } else if cached, err := loadCacheFromDisk(path); err == nil {
size = cached.TotalSize size = cached.TotalSize
} else { } else {
size = calculateDirSizeConcurrent(path, largeFileChan, filesScanned, dirsScanned, bytesScanned, currentPath) size = calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath)
} }
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
@@ -162,7 +163,11 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size, err := getDirectorySizeFromDu(path) size, err := func() (int64, error) {
duSem <- struct{}{}
defer func() { <-duSem }()
return getDirectorySizeFromDu(path)
}()
if err != nil || size <= 0 { if err != nil || size <= 0 {
size = calculateDirSizeFast(path, filesScanned, dirsScanned, bytesScanned, currentPath) size = calculateDirSizeFast(path, filesScanned, dirsScanned, bytesScanned, currentPath)
} }
@@ -186,7 +191,7 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size := calculateDirSizeConcurrent(path, largeFileChan, filesScanned, dirsScanned, bytesScanned, currentPath) size := calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath)
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
@@ -416,7 +421,7 @@ func isInFoldedDir(path string) bool {
return false return false
} }
func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, filesScanned, dirsScanned, bytesScanned *int64, currentPath *string) int64 { func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, duSem chan struct{}, filesScanned, dirsScanned, bytesScanned *int64, currentPath *string) int64 {
children, err := os.ReadDir(root) children, err := os.ReadDir(root)
if err != nil { if err != nil {
return 0 return 0
@@ -451,7 +456,11 @@ func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, fil
go func(path string) { go func(path string) {
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size, err := getDirectorySizeFromDu(path) size, err := func() (int64, error) {
duSem <- struct{}{}
defer func() { <-duSem }()
return getDirectorySizeFromDu(path)
}()
if err == nil && size > 0 { if err == nil && size > 0 {
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(bytesScanned, size) atomic.AddInt64(bytesScanned, size)
@@ -467,7 +476,7 @@ func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, fil
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size := calculateDirSizeConcurrent(path, largeFileChan, filesScanned, dirsScanned, bytesScanned, currentPath) size := calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath)
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
}(fullPath) }(fullPath)