diff --git a/.cargo/config.toml b/.cargo/config.toml index 804c9f9f..32a3e6e1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -11,6 +11,7 @@ linker = "aarch64-linux-gnu-gcc" [target.aarch64-unknown-linux-musl] linker = "aarch64-linux-musl-gcc" +rustflags = ["-C", "target-feature=+crt-static", "-C", "link-arg=-s"] [target.armv7-unknown-linux-gnueabihf] linker = "arm-linux-gnueabihf-gcc" @@ -20,6 +21,7 @@ linker = "arm-linux-musleabihf-gcc" [target.x86_64-unknown-linux-musl] linker = "x86_64-linux-musl-gcc" +rustflags = ["-C", "target-feature=+crt-static", "-C", "link-arg=-s"] [target.powerpc64le-unknown-linux-gnu] linker = "powerpc64le-linux-gnu-gcc" @@ -43,6 +45,8 @@ linker = "riscv64-linux-gnu-gcc" # Build configuration [build] +# Use a shared target directory for all workspace members +target-dir = "target" # Default target intentionally left as host; set --target explicitly (use `cross` for Linux) # Cross-compilation settings (commented out - let cross-rs handle Docker images) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 59decec3..649e36a8 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -22,11 +22,16 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - CARGO_INCREMENTAL: 0 CARGO_NET_RETRY: 10 RUSTUP_MAX_RETRIES: 10 REGISTRY: ghcr.io IMAGE_NAME: terraphim/terraphim-ai + # sccache configuration for distributed caching + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + # Use CI-optimized profiles + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_TEST_DEBUG: 0 jobs: # Build setup and metadata @@ -103,6 +108,9 @@ jobs: components: rustfmt, clippy targets: ${{ matrix.target }} + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Cache Cargo registry and dependencies (self-hosted) uses: actions/cache@v4 with: @@ -129,33 +137,39 @@ jobs: - name: Build release binaries run: | - # Build workspace with default features (no rocksdb for faster CI) - cargo build --release --target ${{ matrix.target }} --workspace + # Start sccache server + sccache --start-server || true + + # Build workspace with default features using CI-optimized profile + cargo build --profile ci-release --target ${{ matrix.target }} --workspace # Verify key binaries exist - ls -la target/${{ matrix.target }}/release/terraphim* + ls -la target/${{ matrix.target }}/ci-release/terraphim* # Show binary sizes - for binary in target/${{ matrix.target }}/release/terraphim*; do + for binary in target/${{ matrix.target }}/ci-release/terraphim*; do if [[ -f "$binary" ]]; then echo "$(basename "$binary"): $(du -h "$binary" | cut -f1)" fi done + + # Show sccache stats + sccache -s || true - name: Run tests run: | - # Run unit and integration tests (exclude integration-signing which requires zipsign CLI) - cargo test --release --target ${{ matrix.target }} --workspace --features "self_update/signatures" + # Run unit and integration tests with CI profile + cargo test --profile ci --target ${{ matrix.target }} --workspace --features "self_update/signatures" - name: Upload binary artifacts uses: actions/upload-artifact@v4 with: name: rust-binaries-${{ matrix.target }} path: | - target/${{ matrix.target }}/release/terraphim_server - target/${{ matrix.target }}/release/terraphim_mcp_server - target/${{ matrix.target }}/release/terraphim-agent - retention-days: ${{ needs.setup.outputs.is-release == 'true' && '90' || '30' }} + target/${{ matrix.target }}/ci-release/terraphim_server + target/${{ matrix.target }}/ci-release/terraphim_mcp_server + target/${{ matrix.target }}/ci-release/terraphim-agent + retention-days: ${{ needs.setup.outputs.is-release == 'true' && '30' || '7' }} - name: Create .deb package if: matrix.target == 'x86_64-unknown-linux-gnu' @@ -177,7 +191,7 @@ jobs: with: name: deb-packages path: target/${{ matrix.target }}/debian/*.deb - retention-days: ${{ needs.setup.outputs.is-release == 'true' && '90' || '30' }} + retention-days: ${{ needs.setup.outputs.is-release == 'true' && '30' || '7' }} # Frontend build frontend-build: @@ -224,7 +238,7 @@ jobs: with: name: frontend-dist path: desktop/dist/ - retention-days: ${{ needs.setup.outputs.is-release == 'true' && '90' || '30' }} + retention-days: ${{ needs.setup.outputs.is-release == 'true' && '30' || '7' }} # WASM build wasm-build: @@ -264,7 +278,7 @@ jobs: with: name: wasm-package path: crates/terraphim_automata/wasm-test/pkg/ - retention-days: ${{ needs.setup.outputs.is-release == 'true' && '90' || '30' }} + retention-days: ${{ needs.setup.outputs.is-release == 'true' && '30' || '7' }} # Docker image build docker-build: diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 00000000..37311e6c --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,75 @@ +name: Nightly Cleanup +on: + schedule: + - cron: '0 2 * * *' # 2 AM daily + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + cleanup: + name: Cleanup Build Artifacts + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Run target cleanup script + run: | + chmod +x scripts/cleanup-target.sh + ./scripts/cleanup-target.sh --retention 3 + + - name: Cleanup old GitHub Actions artifacts + run: | + # Keep only artifacts from last 10 workflow runs + gh run list --limit 50 --json databaseId,createdAt | \ + jq -r '.[10:].databaseId' | \ + xargs -I {} gh run delete {} || true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cleanup cargo cache + run: | + if command -v cargo-cache &> /dev/null; then + echo "Running cargo cache autoclean..." + cargo cache --autoclean + else + echo "cargo-cache not installed, skipping..." + fi + + # Clean up old cargo registry cache manually + if [[ -d ~/.cargo/registry/cache ]]; then + echo "Cleaning old cargo registry cache..." + find ~/.cargo/registry/cache -type f -mtime +7 -delete || true + fi + + - name: Report disk usage + run: | + echo "## Disk Usage Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Current Disk Usage" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + df -h >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [[ -d target ]]; then + echo "### Target Directory Size" >> $GITHUB_STEP_SUMMARY + echo "$(du -sh target 2>/dev/null | cut -f1)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + if [[ -d ~/.cargo/registry ]]; then + echo "### Cargo Registry Size" >> $GITHUB_STEP_SUMMARY + echo "$(du -sh ~/.cargo/registry 2>/dev/null | cut -f1)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + if [[ -d /opt/cargo-cache ]]; then + echo "### Self-Hosted Cache Size" >> $GITHUB_STEP_SUMMARY + echo "$(du -sh /opt/cargo-cache 2>/dev/null | cut -f1)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.gitignore b/.gitignore index 9f84ceed..468b1160 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ keys/ # Claude Code working files .claude/plans/ + +# Build output logs (should not be committed) +cargo_test_output.log +*.log diff --git a/Cargo.toml b/Cargo.toml index aac142b1..f392bed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,15 +30,53 @@ log = "0.4" genai = { git = "https://github.com/terraphim/rust-genai.git", branch = "merge-upstream-20251103" } self_update = { git = "https://github.com/AlexMikhalev/self_update.git", branch = "update-zipsign-api-v0.2" } +# Optimized profiles for disk usage and build performance + +[profile.dev] +# Enable incremental with optimized settings +incremental = true +codegen-units = 256 +# Split debug info to reduce object file sizes +split-debuginfo = "unpacked" + +[profile.test] +incremental = true +codegen-units = 256 +split-debuginfo = "unpacked" + [profile.release] panic = "unwind" -lto = false +lto = "thin" codegen-units = 1 opt-level = 3 +strip = "debuginfo" [profile.release-lto] inherits = "release" lto = true codegen-units = 1 opt-level = 3 +strip = true panic = "abort" + +[profile.size-optimized] +inherits = "release" +opt-level = "z" +lto = true +codegen-units = 1 +strip = true +panic = "abort" + +# CI-optimized profiles for faster builds with less disk usage +[profile.ci] +inherits = "dev" +incremental = false +codegen-units = 16 +split-debuginfo = "off" +debug = false + +[profile.ci-release] +inherits = "release" +lto = "thin" +codegen-units = 8 +strip = "debuginfo" diff --git a/crates/terraphim_automata/src/lib.rs b/crates/terraphim_automata/src/lib.rs index 86f03e91..9208587a 100644 --- a/crates/terraphim_automata/src/lib.rs +++ b/crates/terraphim_automata/src/lib.rs @@ -347,8 +347,24 @@ pub async fn load_thesaurus(automata_path: &AutomataPath) -> Result { } let contents = match automata_path { - AutomataPath::Local(path) => fs::read_to_string(path)?, - AutomataPath::Remote(url) => read_url(url.clone()).await?, + AutomataPath::Local(path) => { + // Read file directly and handle errors atomically (avoid TOCTOU race) + fs::read_to_string(path).map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + TerraphimAutomataError::InvalidThesaurus(format!( + "Thesaurus file not found: {}", + path.display() + )) + } else { + TerraphimAutomataError::Io(e) + } + })? + } + AutomataPath::Remote(_) => { + return Err(TerraphimAutomataError::InvalidThesaurus( + "Remote loading is not supported. Enable the 'remote-loading' feature.".to_string(), + )); + } }; let thesaurus = serde_json::from_str(&contents)?; diff --git a/crates/terraphim_service/src/lib.rs b/crates/terraphim_service/src/lib.rs index 87235fc2..668e9e5e 100644 --- a/crates/terraphim_service/src/lib.rs +++ b/crates/terraphim_service/src/lib.rs @@ -259,11 +259,25 @@ impl TerraphimService { Ok(thesaurus) } Err(e) => { - log::error!( - "Failed to build thesaurus from local KG for role {}: {:?}", - role_name, - e - ); + // Check if error is "file not found" (expected for optional files) + // and downgrade log level from ERROR to DEBUG + let is_file_not_found = + e.to_string().contains("file not found") + || e.to_string().contains("not found:"); + + if is_file_not_found { + log::debug!( + "Failed to build thesaurus from local KG (optional file not found) for role {}: {:?}", + role_name, + e + ); + } else { + log::error!( + "Failed to build thesaurus from local KG for role {}: {:?}", + role_name, + e + ); + } Err(ServiceError::Config( "Failed to load or build thesaurus".into(), )) @@ -345,11 +359,19 @@ impl TerraphimService { Ok(thesaurus) } Err(e) => { - log::error!( - "Failed to build thesaurus from local KG for role {}: {:?}", - role_name, - e - ); + // Check if error is "file not found" (expected for optional files) + // and downgrade log level from ERROR to DEBUG + let is_file_not_found = e.to_string().contains("file not found"); + + if is_file_not_found { + log::debug!("Failed to build thesaurus from local KG (optional file not found) for role {}: {:?}", role_name, e); + } else { + log::error!( + "Failed to build thesaurus from local KG for role {}: {:?}", + role_name, + e + ); + } Err(ServiceError::Config( "Failed to build thesaurus from local KG".into(), )) @@ -417,7 +439,19 @@ impl TerraphimService { rolegraphs.insert(role_name.clone(), rolegraph_value); } Err(e) => { - log::error!("Failed to update role and thesaurus: {:?}", e) + // Check if error is "file not found" (expected for optional files) + // and downgrade log level from ERROR to DEBUG + let is_file_not_found = + e.to_string().contains("file not found"); + + if is_file_not_found { + log::debug!("Failed to update role and thesaurus (optional file not found): {:?}", e); + } else { + log::error!( + "Failed to update role and thesaurus: {:?}", + e + ); + } } } @@ -459,7 +493,16 @@ impl TerraphimService { Ok(thesaurus) } Err(e) => { - log::error!("Failed to load thesaurus: {:?}", e); + // Check if error is "file not found" (expected for optional files) + // and downgrade log level from ERROR to DEBUG + let is_file_not_found = e.to_string().contains("file not found") + || e.to_string().contains("not found:"); + + if is_file_not_found { + log::debug!("Thesaurus file not found (optional): {:?}", e); + } else { + log::error!("Failed to load thesaurus: {:?}", e); + } // Try to build thesaurus from KG and update the config_state directly let mut rolegraphs = self.config_state.roles.clone(); let result = load_thesaurus_from_automata_path( diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index da0429c2..32c41b7a 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,5 +1,5 @@ # Multi-stage base Dockerfile for Terraphim AI -# Optimized for CI/CD with fast builds and layer caching +# Optimized for CI/CD with fast builds, layer caching, and minimal disk usage # ============================================ # Stage 1: Base Builder @@ -12,7 +12,9 @@ ENV CARGO_TERM_COLOR=always \ RUST_BACKTRACE=1 \ RUSTFLAGS="-C target-cpu=generic" \ CARGO_NET_RETRY=10 \ - CARGO_IO_MAX_THREADS=8 + CARGO_IO_MAX_THREADS=8 \ + SCCACHE_DIR=/var/cache/sccache \ + RUSTC_WRAPPER=sccache # Install system dependencies (with proper layering for caching) RUN apt-get update && apt-get install -y \ @@ -29,6 +31,8 @@ RUN apt-get update && apt-get install -y \ unzip \ tar \ gzip \ + # sccache for distributed caching + sccache \ # Clean up after base packages && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -60,7 +64,8 @@ USER terraphim RUN cargo install --version 0.12.2 cargo-edit && \ cargo install --version 0.18.3 cargo-audit && \ cargo install --version 0.14.2 cargo-deny && \ - cargo install --version 8.4.0 cargo-watch || true + cargo install --version 8.4.0 cargo-watch && \ + cargo install --version 0.8.3 cargo-cache || true # ============================================ # Stage 2: Rust Builder with Cache @@ -79,9 +84,12 @@ RUN mkdir -p crates/terraphim_automata/src && \ mkdir -p terraphim_server/src && \ echo "fn main() {}" > terraphim_server/src/main.rs -# Build dependencies (creates cached layers) -RUN cargo build --release --bins --workspace && \ - rm -rf target/release/deps/terraphim* || true +# Build dependencies with cache mounts for optimal disk usage +RUN --mount=type=cache,target=/var/cache/sccache \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo build --profile ci-release --bins --workspace && \ + rm -rf target/ci-release/deps/terraphim* || true # ============================================ # Stage 3: WASM Builder @@ -137,8 +145,8 @@ RUN useradd -m -s /bin/bash terraphim && \ WORKDIR /app # Copy built artifacts from builder stages -COPY --from=rust-builder --chown=terraphim:terraphim /app/target/release/terraphim_server /app/bin/ -COPY --from=rust-builder --chown=terraphim:terraphim /app/target/release/terraphim_mcp_server /app/bin/ +COPY --from=rust-builder --chown=terraphim:terraphim /app/target/ci-release/terraphim_server /app/bin/ +COPY --from=rust-builder --chown=terraphim:terraphim /app/target/ci-release/terraphim_mcp_server /app/bin/ COPY --from=frontend-builder --chown=terraphim:terraphim /app/desktop/dist /app/dist/ # Copy configuration files diff --git a/scripts/build-workspace.sh b/scripts/build-workspace.sh new file mode 100755 index 00000000..c981d8d4 --- /dev/null +++ b/scripts/build-workspace.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# Workspace Build Script with Optimized Settings +# Reduces disk usage while maintaining build performance + +set -euo pipefail + +# Configuration +PROFILE=${PROFILE:-"ci-release"} +TARGET=${TARGET:-"x86_64-unknown-linux-gnu"} +FEATURES=${FEATURES:-""} +CLEAN_FIRST=${CLEAN_FIRST:-"false"} + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +show_help() { + cat << EOF +Terraphim AI Workspace Build Script + +Usage: $0 [OPTIONS] + +OPTIONS: + -p, --profile PROFILE Build profile (dev, release, ci, ci-release, size-optimized) + Default: ci-release + -t, --target TARGET Target triple + Default: x86_64-unknown-linux-gnu + -f, --features FEATURES Comma-separated list of features + -c, --clean Clean before building + -h, --help Show this help message + +PROFILES: + dev Fast builds with debug info + release Optimized release build + ci CI-optimized dev build (faster, less disk) + ci-release CI-optimized release build (default) + size-optimized Maximum size reduction + +EXAMPLES: + # Build with default settings (ci-release profile) + $0 + + # Build with specific features + $0 --features "sqlite,redis" + + # Clean build for release + $0 --profile release --clean + + # Build for different target + $0 --target aarch64-unknown-linux-gnu + +EOF +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -p|--profile) + PROFILE="$2" + shift 2 + ;; + -t|--target) + TARGET="$2" + shift 2 + ;; + -f|--features) + FEATURES="$2" + shift 2 + ;; + -c|--clean) + CLEAN_FIRST="true" + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Validate profile +valid_profiles=("dev" "release" "ci" "ci-release" "size-optimized") +if [[ ! " ${valid_profiles[@]} " =~ " ${PROFILE} " ]]; then + log_error "Invalid profile: $PROFILE" + log_info "Valid profiles: ${valid_profiles[*]}" + exit 1 +fi + +log_step "Building Terraphim AI Workspace" +log_info "Profile: $PROFILE" +log_info "Target: $TARGET" +[[ -n "$FEATURES" ]] && log_info "Features: $FEATURES" + +# Clean if requested +if [[ "$CLEAN_FIRST" == "true" ]]; then + log_step "Cleaning previous builds..." + cargo clean +fi + +# Set environment variables for optimal builds +export CARGO_INCREMENTAL=1 +export CARGO_PROFILE_DEV_CODEGEN_UNITS=256 +export CARGO_PROFILE_DEV_SPLIT_DEBUGINFO=unpacked + +# Build command construction +BUILD_CMD="cargo build --profile $PROFILE --target $TARGET" + +# Add features if specified +if [[ -n "$FEATURES" ]]; then + BUILD_CMD="$BUILD_CMD --features \"$FEATURES\"" +fi + +# Show disk usage before build +log_step "Disk usage before build:" +df -h | grep -E '(Filesystem|/dev/)' || true + +# Build workspace libraries first (parallel friendly) +log_step "Building workspace libraries..." +$BUILD_CMD --workspace --lib + +# Build specific binaries +log_step "Building terraphim_server..." +$BUILD_CMD --package terraphim_server + +log_step "Building terraphim_mcp_server..." +$BUILD_CMD --package terraphim_mcp_server 2>/dev/null || log_warn "terraphim_mcp_server not found" + +log_step "Building terraphim-agent..." +$BUILD_CMD --package terraphim-agent 2>/dev/null || log_warn "terraphim-agent not found" + +# Show binary sizes +log_step "Build complete! Binary sizes:" +TARGET_DIR="target/$TARGET/$PROFILE" +if [[ -d "$TARGET_DIR" ]]; then + for binary in "$TARGET_DIR"/terraphim*; do + if [[ -f "$binary" && -x "$binary" ]]; then + size=$(du -h "$binary" | cut -f1) + name=$(basename "$binary") + log_info "$name: $size" + fi + done +else + log_warn "Target directory not found: $TARGET_DIR" +fi + +# Show disk usage after build +log_step "Disk usage after build:" +df -h | grep -E '(Filesystem|/dev/)' || true + +# Show target directory size +if [[ -d "target" ]]; then + log_info "Target directory size: $(du -sh target 2>/dev/null | cut -f1)" +fi + +log_step "Build completed successfully!" diff --git a/scripts/cleanup-target.sh b/scripts/cleanup-target.sh new file mode 100755 index 00000000..c01c0873 --- /dev/null +++ b/scripts/cleanup-target.sh @@ -0,0 +1,276 @@ +#!/bin/bash +# Target Directory Cleanup Script for Terraphim AI +# Reduces disk usage by removing old build artifacts +# Expected savings: 20-30 GB per runner + +set -euo pipefail + +# Configuration +DRY_RUN=${DRY_RUN:-false} +TARGET_DIR=${TARGET_DIR:-"target"} +CARGO_CACHE_DIR=${CARGO_CACHE_DIR:-"$HOME/.cargo"} +RETENTION_DAYS=${RETENTION_DAYS:-7} +INCREMENTAL_RETENTION_DAYS=${INCREMENTAL_RETENTION_DAYS:-3} + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Show current disk usage +show_disk_usage() { + log_info "Current disk usage:" + df -h | grep -E '(Filesystem|/dev/)' || true + + if [[ -d "$TARGET_DIR" ]]; then + log_info "Target directory size: $(du -sh "$TARGET_DIR" 2>/dev/null | cut -f1 || echo 'N/A')" + fi + + if [[ -d "$CARGO_CACHE_DIR/registry" ]]; then + log_info "Cargo registry size: $(du -sh "$CARGO_CACHE_DIR/registry" 2>/dev/null | cut -f1 || echo 'N/A')" + fi +} + +# Clean old .rlib files +clean_rlibs() { + log_info "Cleaning .rlib files older than $RETENTION_DAYS days..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -name "*.rlib" -type f -mtime +$RETENTION_DAYS -print 2>/dev/null | head -20 + log_warn "Dry run mode - no files deleted" + else + local count=$(find "$TARGET_DIR" -name "*.rlib" -type f -mtime +$RETENTION_DAYS -delete -print 2>/dev/null | wc -l) + log_info "Deleted $count old .rlib files" + fi +} + +# Clean old .rmeta files +clean_rmeta() { + log_info "Cleaning .rmeta files older than $RETENTION_DAYS days..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -name "*.rmeta" -type f -mtime +$RETENTION_DAYS -print 2>/dev/null | head -20 + log_warn "Dry run mode - no files deleted" + else + local count=$(find "$TARGET_DIR" -name "*.rmeta" -type f -mtime +$RETENTION_DAYS -delete -print 2>/dev/null | wc -l) + log_info "Deleted $count old .rmeta files" + fi +} + +# Clean incremental compilation data +clean_incremental() { + log_info "Cleaning incremental compilation data older than $INCREMENTAL_RETENTION_DAYS days..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -path "*/incremental/*" -type d -mtime +$INCREMENTAL_RETENTION_DAYS -print 2>/dev/null | head -20 + log_warn "Dry run mode - no directories deleted" + else + local count=0 + while IFS= read -r dir; do + if [[ -d "$dir" ]]; then + rm -rf "$dir" + ((count++)) + fi + done < <(find "$TARGET_DIR" -path "*/incremental/*" -type d -mtime +$INCREMENTAL_RETENTION_DAYS 2>/dev/null) + log_info "Deleted $count incremental directories" + fi +} + +# Clean object files +clean_objects() { + log_info "Cleaning object files (.o)..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -name "*.o" -type f -print 2>/dev/null | head -20 + log_warn "Dry run mode - no files deleted" + else + local count=$(find "$TARGET_DIR" -name "*.o" -type f -delete -print 2>/dev/null | wc -l) + log_info "Deleted $count object files" + fi +} + +# Clean dependency files +clean_deps() { + log_info "Cleaning dependency files (.d) older than 1 day..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -name "*.d" -type f -mtime +1 -print 2>/dev/null | head -20 + log_warn "Dry run mode - no files deleted" + else + local count=$(find "$TARGET_DIR" -name "*.d" -type f -mtime +1 -delete -print 2>/dev/null | wc -l) + log_info "Deleted $count dependency files" + fi +} + +# Clean old build artifacts for specific targets +clean_old_targets() { + log_info "Cleaning old target-specific artifacts..." + + # List of targets to clean (keep only recent builds) + local targets=("x86_64-unknown-linux-gnu" "aarch64-unknown-linux-gnu" "x86_64-unknown-linux-musl") + + for target in "${targets[@]}"; do + local target_path="$TARGET_DIR/$target" + if [[ -d "$target_path" ]]; then + log_info "Checking $target_path..." + + # Clean old .rlib files in target directory + if [[ "$DRY_RUN" == "true" ]]; then + find "$target_path" -name "*.rlib" -type f -mtime +$RETENTION_DAYS -print 2>/dev/null | head -10 + else + local count=$(find "$target_path" -name "*.rlib" -type f -mtime +$RETENTION_DAYS -delete -print 2>/dev/null | wc -l) + if [[ $count -gt 0 ]]; then + log_info "Deleted $count old .rlib files from $target" + fi + fi + fi + done +} + +# Clean cargo registry cache +clean_cargo_cache() { + log_info "Cleaning Cargo registry cache..." + + if command -v cargo-cache &> /dev/null; then + if [[ "$DRY_RUN" == "true" ]]; then + log_info "Would run: cargo cache --autoclean" + else + cargo cache --autoclean 2>/dev/null || log_warn "cargo cache autoclean failed" + fi + else + log_warn "cargo-cache not installed, skipping registry cleanup" + log_info "Install with: cargo install cargo-cache" + fi +} + +# Remove empty directories +remove_empty_dirs() { + log_info "Removing empty directories..." + + if [[ "$DRY_RUN" == "true" ]]; then + find "$TARGET_DIR" -type d -empty -print 2>/dev/null | head -20 + log_warn "Dry run mode - no directories deleted" + else + local count=0 + # Keep running until no more empty directories are found + while true; do + local deleted=$(find "$TARGET_DIR" -type d -empty -delete -print 2>/dev/null | wc -l) + if [[ $deleted -eq 0 ]]; then + break + fi + ((count+=deleted)) + done + log_info "Removed $count empty directories" + fi +} + +# Show help +show_help() { + cat << EOF +Terraphim AI Target Directory Cleanup Script + +Usage: $0 [OPTIONS] + +OPTIONS: + -d, --dry-run Show what would be deleted without deleting + -t, --target-dir DIR Target directory to clean (default: target) + -r, --retention DAYS Retention period in days (default: 7) + -h, --help Show this help message + +ENVIRONMENT VARIABLES: + DRY_RUN Set to 'true' for dry run mode + TARGET_DIR Target directory path + CARGO_CACHE_DIR Cargo cache directory path + RETENTION_DAYS Number of days to retain files + INCREMENTAL_RETENTION_DAYS Number of days to retain incremental data + +EXAMPLES: + # Dry run to see what would be deleted + $0 --dry-run + + # Clean with custom retention period + $0 --retention 3 + + # Clean specific target directory + $0 --target-dir /path/to/target + +EOF +} + +# Main function +main() { + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + -d|--dry-run) + DRY_RUN=true + shift + ;; + -t|--target-dir) + TARGET_DIR="$2" + shift 2 + ;; + -r|--retention) + RETENTION_DAYS="$2" + shift 2 + ;; + -h|--help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + log_info "Starting target directory cleanup..." + log_info "Target directory: $TARGET_DIR" + log_info "Retention period: $RETENTION_DAYS days" + log_info "Incremental retention: $INCREMENTAL_RETENTION_DAYS days" + + if [[ "$DRY_RUN" == "true" ]]; then + log_warn "DRY RUN MODE - No files will be deleted" + fi + + # Show initial disk usage + show_disk_usage + + # Perform cleanup + if [[ -d "$TARGET_DIR" ]]; then + clean_rlibs + clean_rmeta + clean_incremental + clean_objects + clean_deps + clean_old_targets + remove_empty_dirs + else + log_warn "Target directory not found: $TARGET_DIR" + fi + + # Clean cargo cache + clean_cargo_cache + + # Show final disk usage + log_info "Cleanup complete!" + show_disk_usage +} + +# Run main function +main "$@"