mirrors/Mole
1
0
Fork 0
mirror of https://github.com/tw93/Mole.git synced 2026-02-04 15:04:42 +00:00
Code Activity

fix(analyze): Additional improvements to limit sem blocking

Browse Source
This commit is contained in:
Jack Phallen
2026-01-11 23:28:44 -05:00
parent 91fcbb925b
commit 1e2cf97b47
1 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/analyze/scanner.go
View File

@@ -50,7 +50,8 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
numWorkers = 1 numWorkers = 1
} }
sem := make(chan struct{}, numWorkers) sem := make(chan struct{}, numWorkers)
duSem := make(chan struct{}, min(4, runtime.NumCPU())) duSem := make(chan struct{}, min(4, runtime.NumCPU())) // limits concurrent du processes
duQueueSem := make(chan struct{}, min(4, runtime.NumCPU())*2) // limits how many goroutines may be waiting to run du
var wg sync.WaitGroup var wg sync.WaitGroup
// Collect results via channels. // Collect results via channels.
@@ -147,7 +148,7 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
} else if cached, err := loadCacheFromDisk(path); err == nil { } else if cached, err := loadCacheFromDisk(path); err == nil {
size = cached.TotalSize size = cached.TotalSize
} else { } else {
size = calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath) size = calculateDirSizeConcurrent(path, largeFileChan, duSem, duQueueSem, filesScanned, dirsScanned, bytesScanned, currentPath)
} }
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
@@ -165,11 +166,11 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
// Folded dirs: fast size without expanding. // Folded dirs: fast size without expanding.
if shouldFoldDirWithPath(child.Name(), fullPath) { if shouldFoldDirWithPath(child.Name(), fullPath) {
sem <- struct{}{} duQueueSem <- struct{}{}
wg.Add(1) wg.Add(1)
go func(name, path string) { go func(name, path string) {
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-duQueueSem }()
size, err := func() (int64, error) { size, err := func() (int64, error) {
duSem <- struct{}{} duSem <- struct{}{}
@@ -199,7 +200,7 @@ func scanPathConcurrent(root string, filesScanned, dirsScanned, bytesScanned *in
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size := calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath) size := calculateDirSizeConcurrent(path, largeFileChan, duSem, duQueueSem, filesScanned, dirsScanned, bytesScanned, currentPath)
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
@@ -429,7 +430,7 @@ func isInFoldedDir(path string) bool {
return false return false
} }
func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, duSem chan struct{}, filesScanned, dirsScanned, bytesScanned *int64, currentPath *string) int64 { func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, duSem, duQueueSem chan struct{}, filesScanned, dirsScanned, bytesScanned *int64, currentPath *string) int64 {
children, err := os.ReadDir(root) children, err := os.ReadDir(root)
if err != nil { if err != nil {
return 0 return 0
@@ -459,11 +460,12 @@ func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, duS
if child.IsDir() { if child.IsDir() {
if shouldFoldDirWithPath(child.Name(), fullPath) { if shouldFoldDirWithPath(child.Name(), fullPath) {
sem <- struct{}{} duQueueSem <- struct{}{}
wg.Add(1) wg.Add(1)
go func(path string) { go func(path string) {
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-duQueueSem }()
size, err := func() (int64, error) { size, err := func() (int64, error) {
duSem <- struct{}{} duSem <- struct{}{}
defer func() { <-duSem }() defer func() { <-duSem }()
@@ -484,7 +486,7 @@ func calculateDirSizeConcurrent(root string, largeFileChan chan<- fileEntry, duS
defer wg.Done() defer wg.Done()
defer func() { <-sem }() defer func() { <-sem }()
size := calculateDirSizeConcurrent(path, largeFileChan, duSem, filesScanned, dirsScanned, bytesScanned, currentPath) size := calculateDirSizeConcurrent(path, largeFileChan, duSem, duQueueSem, filesScanned, dirsScanned, bytesScanned, currentPath)
atomic.AddInt64(&total, size) atomic.AddInt64(&total, size)
atomic.AddInt64(dirsScanned, 1) atomic.AddInt64(dirsScanned, 1)
}(fullPath) }(fullPath)