1
0
mirror of https://github.com/tw93/Mole.git synced 2026-03-23 18:45:08 +00:00

fix(clean): harden project cache scans

Refs #541
This commit is contained in:
Tw93
2026-03-06 07:49:44 +08:00
parent 7e69a4eb71
commit 92ad46a396
4 changed files with 361 additions and 166 deletions

View File

@@ -1,6 +1,9 @@
#!/bin/bash #!/bin/bash
# Cache Cleanup Module # Cache Cleanup Module
set -euo pipefail set -euo pipefail
# shellcheck disable=SC1091
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/purge_shared.sh"
# Preflight TCC prompts once to avoid mid-run interruptions. # Preflight TCC prompts once to avoid mid-run interruptions.
check_tcc_permissions() { check_tcc_permissions() {
[[ -t 1 ]] || return 0 [[ -t 1 ]] || return 0
@@ -88,154 +91,138 @@ clean_service_worker_cache() {
fi fi
fi fi
} }
# Next.js/Python project caches with tight scan bounds and timeouts. # Check whether a directory looks like a project container.
clean_project_caches() { project_cache_has_indicators() {
stop_inline_spinner 2> /dev/null || true local dir="$1"
# Fast pre-check before scanning the whole home dir. local max_depth="${2:-5}"
local has_dev_projects=false local indicator_timeout="${MOLE_PROJECT_CACHE_DISCOVERY_TIMEOUT:-2}"
local -a common_dev_dirs=( [[ -d "$dir" ]] || return 1
"$HOME/Code"
"$HOME/Projects" local -a find_args=("$dir" "-maxdepth" "$max_depth" "(")
"$HOME/workspace" local first=true
"$HOME/github" local indicator
"$HOME/dev" for indicator in "${MOLE_PURGE_PROJECT_INDICATORS[@]}"; do
"$HOME/work" if [[ "$first" == "true" ]]; then
"$HOME/src" first=false
"$HOME/repos" else
"$HOME/Developer" find_args+=("-o")
"$HOME/Development" fi
"$HOME/www" find_args+=("-name" "$indicator")
"$HOME/golang" done
"$HOME/go" find_args+=(")" "-print" "-quit")
"$HOME/rust"
"$HOME/python" run_with_timeout "$indicator_timeout" find "${find_args[@]}" 2> /dev/null | grep -q .
"$HOME/ruby" }
"$HOME/java"
"$HOME/dotnet" # Discover candidate project roots without scanning the whole home directory.
"$HOME/node" discover_project_cache_roots() {
) local -a roots=()
for dir in "${common_dev_dirs[@]}"; do local root
if [[ -d "$dir" ]]; then
has_dev_projects=true for root in "${MOLE_PURGE_DEFAULT_SEARCH_PATHS[@]}"; do
break [[ -d "$root" ]] && roots+=("$root")
done
while IFS= read -r root; do
[[ -d "$root" ]] && roots+=("$root")
done < <(mole_purge_read_paths_config "$HOME/.config/mole/purge_paths")
local dir
local base
for dir in "$HOME"/*/; do
[[ -d "$dir" ]] || continue
dir="${dir%/}"
base=$(basename "$dir")
case "$base" in
.* | Library | Applications | Movies | Music | Pictures | Public)
continue
;;
esac
if project_cache_has_indicators "$dir" 5; then
roots+=("$dir")
fi fi
done done
# Fallback: look for project markers near $HOME.
if [[ "$has_dev_projects" == "false" ]]; then [[ ${#roots[@]} -eq 0 ]] && return 0
local -a project_markers=(
"node_modules" printf '%s\n' "${roots[@]}" | LC_ALL=C sort -u
".git" }
"target"
"go.mod" # Scan a project root for supported build caches while pruning heavy subtrees.
"Cargo.toml" scan_project_cache_root() {
"package.json" local root="$1"
"pom.xml" local output_file="$2"
"build.gradle" local scan_timeout="${MOLE_PROJECT_CACHE_SCAN_TIMEOUT:-6}"
) [[ -d "$root" ]] || return 0
local spinner_active=false
if [[ -t 1 ]]; then local -a find_args=(
MOLE_SPINNER_PREFIX=" " find -P "$root" -maxdepth 9 -mount
start_inline_spinner "Detecting dev projects..." "(" -name "Library" -o -name ".Trash" -o -name "node_modules" -o -name ".git" -o -name ".svn" -o -name ".hg" -o -name ".venv" -o -name "venv" -o -name ".pnpm-store" -o -name ".fvm" -o -name "DerivedData" -o -name "Pods" ")"
spinner_active=true -prune -o
fi -type d
for marker in "${project_markers[@]}"; do "(" -name ".next" -o -name "__pycache__" -o -name ".dart_tool" ")"
if run_with_timeout 3 sh -c "find '$HOME' -maxdepth 2 -name '$marker' -not -path '*/Library/*' -not -path '*/.Trash/*' 2>/dev/null | head -1" | grep -q .; then -print
has_dev_projects=true )
break
fi local status=0
done run_with_timeout "$scan_timeout" "${find_args[@]}" >> "$output_file" 2> /dev/null || status=$?
if [[ "$spinner_active" == "true" ]]; then
stop_inline_spinner 2> /dev/null || true if [[ $status -eq 124 ]]; then
# Extra clear to prevent spinner character remnants in terminal debug_log "Project cache scan timed out: $root"
[[ -t 1 ]] && printf "\r\033[2K" >&2 || true elif [[ $status -ne 0 ]]; then
fi debug_log "Project cache scan failed (${status}): $root"
[[ "$has_dev_projects" == "false" ]] && return 0
fi fi
return 0
}
# Next.js/Python/Flutter project caches scoped to discovered project roots.
clean_project_caches() {
stop_inline_spinner 2> /dev/null || true
local matches_tmp_file
matches_tmp_file=$(create_temp_file)
local -a scan_roots=()
local root
while IFS= read -r root; do
[[ -n "$root" ]] && scan_roots+=("$root")
done < <(discover_project_cache_roots)
[[ ${#scan_roots[@]} -eq 0 ]] && return 0
if [[ -t 1 ]]; then if [[ -t 1 ]]; then
MOLE_SPINNER_PREFIX=" " MOLE_SPINNER_PREFIX=" "
start_inline_spinner "Searching project caches..." start_inline_spinner "Searching project caches..."
fi fi
local nextjs_tmp_file
nextjs_tmp_file=$(create_temp_file) for root in "${scan_roots[@]}"; do
local pycache_tmp_file scan_project_cache_root "$root" "$matches_tmp_file"
pycache_tmp_file=$(create_temp_file)
local flutter_tmp_file
flutter_tmp_file=$(create_temp_file)
local find_timeout=30
# Parallel scans (Next.js and __pycache__).
# Note: -maxdepth must come before -name for BSD find compatibility
(
command find -P "$HOME" -maxdepth 3 -mount -type d -name ".next" \
-not -path "*/Library/*" \
-not -path "*/.Trash/*" \
-not -path "*/node_modules/*" \
-not -path "*/.*" \
2> /dev/null || true
) > "$nextjs_tmp_file" 2>&1 &
local next_pid=$!
(
command find -P "$HOME" -maxdepth 3 -mount -type d -name "__pycache__" \
-not -path "*/Library/*" \
-not -path "*/.Trash/*" \
-not -path "*/node_modules/*" \
-not -path "*/.*" \
2> /dev/null || true
) > "$pycache_tmp_file" 2>&1 &
local py_pid=$!
(
command find -P "$HOME" -maxdepth 5 -mount -type d -name ".dart_tool" \
-not -path "*/Library/*" \
-not -path "*/.Trash/*" \
-not -path "*/node_modules/*" \
-not -path "*/.fvm/*" \
2> /dev/null || true
) > "$flutter_tmp_file" 2>&1 &
local flutter_pid=$!
local elapsed=0
local check_interval=0.2 # Check every 200ms instead of 1s for smoother experience
while [[ $(echo "$elapsed < $find_timeout" | awk '{print ($1 < $2)}') -eq 1 ]]; do
if ! kill -0 $next_pid 2> /dev/null && ! kill -0 $py_pid 2> /dev/null && ! kill -0 $flutter_pid 2> /dev/null; then
break
fi
sleep $check_interval
elapsed=$(echo "$elapsed + $check_interval" | awk '{print $1 + $2}')
done
# Kill stuck scans after timeout.
for pid in $next_pid $py_pid $flutter_pid; do
if kill -0 "$pid" 2> /dev/null; then
kill -TERM "$pid" 2> /dev/null || true
local grace_period=0
while [[ $grace_period -lt 20 ]]; do
if ! kill -0 "$pid" 2> /dev/null; then
break
fi
sleep 0.1
grace_period=$((grace_period + 1))
done
if kill -0 "$pid" 2> /dev/null; then
kill -KILL "$pid" 2> /dev/null || true
fi
wait "$pid" 2> /dev/null || true
else
wait "$pid" 2> /dev/null || true
fi
done done
if [[ -t 1 ]]; then if [[ -t 1 ]]; then
stop_inline_spinner stop_inline_spinner
fi fi
while IFS= read -r next_dir; do
[[ -d "$next_dir/cache" ]] && safe_clean "$next_dir/cache"/* "Next.js build cache" || true while IFS= read -r cache_dir; do
done < "$nextjs_tmp_file" case "$(basename "$cache_dir")" in
while IFS= read -r pycache; do ".next")
[[ -d "$pycache" ]] && safe_clean "$pycache"/* "Python bytecode cache" || true [[ -d "$cache_dir/cache" ]] && safe_clean "$cache_dir/cache"/* "Next.js build cache" || true
done < "$pycache_tmp_file" ;;
while IFS= read -r flutter_tool; do "__pycache__")
if [[ -d "$flutter_tool" ]]; then [[ -d "$cache_dir" ]] && safe_clean "$cache_dir"/* "Python bytecode cache" || true
safe_clean "$flutter_tool" "Flutter build cache (.dart_tool)" || true ;;
local build_dir="$(dirname "$flutter_tool")/build" ".dart_tool")
if [[ -d "$build_dir" ]]; then if [[ -d "$cache_dir" ]]; then
safe_clean "$build_dir" "Flutter build cache (build/)" || true safe_clean "$cache_dir" "Flutter build cache (.dart_tool)" || true
fi local build_dir="$(dirname "$cache_dir")/build"
fi if [[ -d "$build_dir" ]]; then
done < "$flutter_tmp_file" safe_clean "$build_dir" "Flutter build cache (build/)" || true
fi
fi
;;
esac
done < <(LC_ALL=C sort -u "$matches_tmp_file" 2> /dev/null)
} }

View File

@@ -20,12 +20,18 @@ readonly MOLE_TIMEOUT_LOADED=1
# Recommendation: Install coreutils for reliable timeout support # Recommendation: Install coreutils for reliable timeout support
# brew install coreutils # brew install coreutils
# #
# Fallback order:
# 1. gtimeout / timeout
# 2. perl helper with dedicated process group cleanup
# 3. shell-based fallback (last resort)
#
# The shell-based fallback has known limitations: # The shell-based fallback has known limitations:
# - May not clean up all child processes # - May not clean up all child processes
# - Has race conditions in edge cases # - Has race conditions in edge cases
# - Less reliable than native timeout command # - Less reliable than native timeout/perl helper
if [[ -z "${MO_TIMEOUT_INITIALIZED:-}" ]]; then if [[ -z "${MO_TIMEOUT_INITIALIZED:-}" ]]; then
MO_TIMEOUT_BIN="" MO_TIMEOUT_BIN=""
MO_TIMEOUT_PERL_BIN=""
for candidate in gtimeout timeout; do for candidate in gtimeout timeout; do
if command -v "$candidate" > /dev/null 2>&1; then if command -v "$candidate" > /dev/null 2>&1; then
MO_TIMEOUT_BIN="$candidate" MO_TIMEOUT_BIN="$candidate"
@@ -36,8 +42,15 @@ if [[ -z "${MO_TIMEOUT_INITIALIZED:-}" ]]; then
fi fi
done done
if [[ -z "$MO_TIMEOUT_BIN" ]] && command -v perl > /dev/null 2>&1; then
MO_TIMEOUT_PERL_BIN="$(command -v perl)"
if [[ "${MO_DEBUG:-0}" == "1" ]]; then
echo "[TIMEOUT] Using perl fallback: $MO_TIMEOUT_PERL_BIN" >&2
fi
fi
# Log warning if no timeout command available # Log warning if no timeout command available
if [[ -z "$MO_TIMEOUT_BIN" ]] && [[ "${MO_DEBUG:-0}" == "1" ]]; then if [[ -z "$MO_TIMEOUT_BIN" && -z "$MO_TIMEOUT_PERL_BIN" ]] && [[ "${MO_DEBUG:-0}" == "1" ]]; then
echo "[TIMEOUT] No timeout command found, using shell fallback" >&2 echo "[TIMEOUT] No timeout command found, using shell fallback" >&2
echo "[TIMEOUT] Install coreutils for better reliability: brew install coreutils" >&2 echo "[TIMEOUT] Install coreutils for better reliability: brew install coreutils" >&2
fi fi
@@ -95,6 +108,66 @@ run_with_timeout() {
return $? return $?
fi fi
# Use perl helper when timeout command is unavailable.
if [[ -n "${MO_TIMEOUT_PERL_BIN:-}" ]]; then
if [[ "${MO_DEBUG:-0}" == "1" ]]; then
echo "[TIMEOUT] Perl fallback, ${duration}s: $*" >&2
fi
"$MO_TIMEOUT_PERL_BIN" -e '
use strict;
use warnings;
use POSIX qw(:sys_wait_h setsid);
use Time::HiRes qw(time sleep);
my $duration = 0 + shift @ARGV;
$duration = 1 if $duration <= 0;
my $pid = fork();
defined $pid or exit 125;
if ($pid == 0) {
setsid() or exit 125;
exec @ARGV;
exit 127;
}
my $deadline = time() + $duration;
while (1) {
my $result = waitpid($pid, WNOHANG);
if ($result == $pid) {
if (WIFEXITED($?)) {
exit WEXITSTATUS($?);
}
if (WIFSIGNALED($?)) {
exit 128 + WTERMSIG($?);
}
exit 1;
}
if (time() >= $deadline) {
kill "TERM", -$pid;
sleep 0.5;
for (1 .. 6) {
$result = waitpid($pid, WNOHANG);
if ($result == $pid) {
exit 124;
}
sleep 0.25;
}
kill "KILL", -$pid;
waitpid($pid, 0);
exit 124;
}
sleep 0.1;
}
' "$duration" "$@"
return $?
fi
# ======================================================================== # ========================================================================
# Shell-based fallback implementation # Shell-based fallback implementation
# ======================================================================== # ========================================================================

View File

@@ -119,11 +119,13 @@ setup() {
} }
@test "clean_project_caches completes without errors" { @test "clean_project_caches completes without errors" {
mkdir -p "$HOME/projects/test-app/.next/cache" mkdir -p "$HOME/Projects/test-app/.next/cache"
mkdir -p "$HOME/projects/python-app/__pycache__" mkdir -p "$HOME/Projects/python-app/__pycache__"
touch "$HOME/projects/test-app/.next/cache/test.cache" touch "$HOME/Projects/test-app/package.json"
touch "$HOME/projects/python-app/__pycache__/module.pyc" touch "$HOME/Projects/python-app/pyproject.toml"
touch "$HOME/Projects/test-app/.next/cache/test.cache"
touch "$HOME/Projects/python-app/__pycache__/module.pyc"
run bash -c " run bash -c "
export DRY_RUN=true export DRY_RUN=true
@@ -133,39 +135,142 @@ setup() {
" "
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
rm -rf "$HOME/projects" rm -rf "$HOME/Projects"
} }
@test "clean_project_caches handles timeout gracefully" { @test "clean_project_caches scans configured roots instead of HOME" {
if ! command -v gtimeout >/dev/null 2>&1 && ! command -v timeout >/dev/null 2>&1; then mkdir -p "$HOME/.config/mole"
skip "gtimeout/timeout not available" mkdir -p "$HOME/CustomProjects/app/.next/cache"
touch "$HOME/CustomProjects/app/package.json"
local fake_bin
fake_bin="$(mktemp -d "$HOME/find-bin.XXXXXX")"
local find_log="$HOME/find.log"
cat > "$fake_bin/find" <<EOF
#!/bin/bash
printf '%s\n' "\$*" >> "$find_log"
root=""
prev=""
for arg in "\$@"; do
if [[ "\$prev" == "-P" ]]; then
root="\$arg"
break
fi fi
prev="\$arg"
done
if [[ "\$root" == "$HOME/CustomProjects" ]]; then
printf '%s\n' "$HOME/CustomProjects/app/.next"
fi
EOF
chmod +x "$fake_bin/find"
mkdir -p "$HOME/test-project/.next" run env HOME="$HOME" PROJECT_ROOT="$PROJECT_ROOT" PATH="$fake_bin:$PATH" bash --noprofile --norc <<'EOF'
set -euo pipefail
printf '%s\n' "$HOME/CustomProjects" > "$HOME/.config/mole/purge_paths"
source "$PROJECT_ROOT/lib/core/common.sh"
source "$PROJECT_ROOT/lib/clean/caches.sh"
run_with_timeout() { shift; "$@"; }
safe_clean() { echo "$2|$1"; }
clean_project_caches
EOF
[ "$status" -eq 0 ]
[[ "$output" == *"Next.js build cache"* ]]
grep -q -- "-P $HOME/CustomProjects " "$find_log"
! grep -q -- "-P $HOME " "$find_log"
function find() { rm -rf "$HOME/CustomProjects" "$HOME/.config/mole" "$fake_bin" "$find_log"
sleep 2 # Simulate slow find }
echo "$HOME/test-project/.next"
}
export -f find
timeout_cmd="timeout" @test "clean_project_caches auto-detects top-level project containers" {
command -v timeout >/dev/null 2>&1 || timeout_cmd="gtimeout" mkdir -p "$HOME/go/src/demo/.next/cache"
touch "$HOME/go/src/demo/go.mod"
touch "$HOME/go/src/demo/.next/cache/test.cache"
run $timeout_cmd 15 bash -c " run env HOME="$HOME" PROJECT_ROOT="$PROJECT_ROOT" bash --noprofile --norc <<'EOF'
source '$PROJECT_ROOT/lib/core/common.sh' set -euo pipefail
source '$PROJECT_ROOT/lib/clean/caches.sh' source "$PROJECT_ROOT/lib/core/common.sh"
clean_project_caches source "$PROJECT_ROOT/lib/clean/caches.sh"
" safe_clean() { echo "$2|$1"; }
[ "$status" -eq 0 ] || [ "$status" -eq 124 ] clean_project_caches
EOF
[ "$status" -eq 0 ]
[[ "$output" == *"Next.js build cache|$HOME/go/src/demo/.next/cache/test.cache"* ]]
rm -rf "$HOME/test-project" rm -rf "$HOME/go"
}
@test "clean_project_caches auto-detects nested GOPATH-style project containers" {
mkdir -p "$HOME/go/src/github.com/example/demo/.next/cache"
touch "$HOME/go/src/github.com/example/demo/go.mod"
touch "$HOME/go/src/github.com/example/demo/.next/cache/test.cache"
run env HOME="$HOME" PROJECT_ROOT="$PROJECT_ROOT" bash --noprofile --norc <<'EOF'
set -euo pipefail
source "$PROJECT_ROOT/lib/core/common.sh"
source "$PROJECT_ROOT/lib/clean/caches.sh"
safe_clean() { echo "$2|$1"; }
clean_project_caches
EOF
[ "$status" -eq 0 ]
[[ "$output" == *"Next.js build cache|$HOME/go/src/github.com/example/demo/.next/cache/test.cache"* ]]
rm -rf "$HOME/go"
}
@test "clean_project_caches skips stalled root scans" {
mkdir -p "$HOME/.config/mole"
mkdir -p "$HOME/SlowProjects/app"
printf '%s\n' "$HOME/SlowProjects" > "$HOME/.config/mole/purge_paths"
local fake_bin
fake_bin="$(mktemp -d "$HOME/find-timeout.XXXXXX")"
cat > "$fake_bin/find" <<EOF
#!/bin/bash
root=""
prev=""
for arg in "\$@"; do
if [[ "\$prev" == "-P" ]]; then
root="\$arg"
break
fi
prev="\$arg"
done
if [[ "\$root" == "$HOME/SlowProjects" ]]; then
trap "" TERM
sleep 30
exit 0
fi
exit 0
EOF
chmod +x "$fake_bin/find"
run /usr/bin/perl -e 'alarm 8; exec @ARGV' env -i HOME="$HOME" PROJECT_ROOT="$PROJECT_ROOT" PATH="$fake_bin:$PATH:/usr/bin:/bin:/usr/sbin:/sbin" TERM="${TERM:-xterm-256color}" bash --noprofile --norc <<'EOF'
set -euo pipefail
source "$PROJECT_ROOT/lib/core/common.sh"
source "$PROJECT_ROOT/lib/clean/caches.sh"
MO_TIMEOUT_BIN=""
export MOLE_PROJECT_CACHE_DISCOVERY_TIMEOUT=0.5
export MOLE_PROJECT_CACHE_SCAN_TIMEOUT=0.5
SECONDS=0
clean_project_caches
echo "ELAPSED=$SECONDS"
EOF
[ "$status" -eq 0 ]
[[ "$output" == *"ELAPSED="* ]]
elapsed=$(printf '%s\n' "$output" | awk -F= '/ELAPSED=/{print $2}' | tail -1)
[[ "$elapsed" =~ ^[0-9]+$ ]]
(( elapsed < 5 ))
rm -rf "$HOME/.config/mole" "$HOME/SlowProjects" "$fake_bin"
} }
@test "clean_project_caches excludes Library and Trash directories" { @test "clean_project_caches excludes Library and Trash directories" {
mkdir -p "$HOME/Library/.next/cache" mkdir -p "$HOME/Library/.next/cache"
mkdir -p "$HOME/.Trash/.next/cache" mkdir -p "$HOME/.Trash/.next/cache"
mkdir -p "$HOME/projects/.next/cache" mkdir -p "$HOME/Projects/app/.next/cache"
touch "$HOME/Projects/app/package.json"
run bash -c " run bash -c "
export DRY_RUN=true export DRY_RUN=true
@@ -175,5 +280,5 @@ setup() {
" "
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
rm -rf "$HOME/projects" rm -rf "$HOME/Projects"
} }

View File

@@ -98,6 +98,36 @@ setup() {
[[ "$result" == "timeout_works" ]] [[ "$result" == "timeout_works" ]]
} }
@test "run_with_timeout perl fallback stops TERM-ignoring commands" {
local fake_dir="$BATS_TEST_TMPDIR/timeout-bin"
mkdir -p "$fake_dir"
local fake_cmd="$fake_dir/hang.sh"
cat > "$fake_cmd" <<'EOF'
#!/bin/bash
trap "" TERM
sleep 30
EOF
chmod +x "$fake_cmd"
run /usr/bin/perl -e 'alarm 8; exec @ARGV' env FAKE_CMD="$fake_cmd" bash --noprofile --norc <<'EOF'
set -euo pipefail
source "$PROJECT_ROOT/lib/core/timeout.sh"
MO_TIMEOUT_BIN=""
SECONDS=0
set +e
run_with_timeout 1 "$FAKE_CMD"
status=$?
set -e
echo "STATUS=$status ELAPSED=$SECONDS"
EOF
[ "$status" -eq 0 ]
[[ "$output" == *"STATUS=124"* ]]
elapsed=$(printf '%s\n' "$output" | awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ELAPSED=/) {split($i, kv, "="); print kv[2]}}' | tail -1)
[[ "$elapsed" =~ ^[0-9]+$ ]]
(( elapsed < 6 ))
}
@test "empty version string is handled gracefully" { @test "empty version string is handled gracefully" {
result=$(bash -c ' result=$(bash -c '
latest="" latest=""