Refactor: upgrade deps, improve reliability, update docs and tooling

.github/workflows/ci.yml

@@ -14,15 +14,13 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.24'
+          go-version: '1.26'
 
       - name: Download dependencies
         run: go mod download
 
       - name: Run tests
-        # Note: -race disabled due to pre-existing race conditions in tests that spawn
-        # background goroutines. These should be fixed by adding proper synchronization.
-        run: go test -v ./...
+        run: go test -race -v ./...
 
   lint:
     runs-on: ubuntu-latest
@@ -31,12 +29,12 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.24'
+          go-version: '1.26'
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v7
         with:
-          version: latest
+          version: v2.9.0
 
   security:
     runs-on: ubuntu-latest
@@ -45,16 +43,20 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.24'
+          go-version: '1.26'
+
+      - name: Install gosec
+        run: go install github.com/securego/gosec/v2/cmd/gosec@latest
 
       - name: Run gosec
-        uses: securego/gosec@master
-        with:
-          # Exclude G101 (hardcoded credentials - false positive on env var names)
-          # Exclude G115 (integer overflow - false positive for PR numbers)
-          # Exclude G304 (file inclusion - intentional for CLI tools)
-          # Exclude G306 (file permissions - config files don't need 0600)
-          args: -exclude=G101,G115,G304,G306 ./...
+        # Individual false positives are suppressed with inline #nosec comments.
+        # Global exclusions for rules that produce widespread false positives:
+        #   G115: integer overflow on safe int<->int64 casts
+        #   G703: path traversal taint (CLI tools intentionally read user-supplied paths)
+        #   G704: SSRF taint (HTTP clients call known GitHub/Slack APIs)
+        #   G705: XSS taint (CLI tools print to stderr, not web responses)
+        #   G706: log injection taint (structured slog logger, not raw string concat)
+        run: gosec -exclude=G115,G703,G704,G705,G706 ./...
 
   build:
     runs-on: ubuntu-latest
@@ -64,17 +66,35 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.24'
+          go-version: '1.26'
 
       - name: Build
         run: go build -v ./...
 
+  scan:
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          severity: 'CRITICAL,HIGH'
+          exit-code: '1'
+
   deploy:
     runs-on: ubuntu-latest
-    needs: [build, security]
+    needs: [build, security, scan]
     # Only deploy on push to main (not on PRs)
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
+    environment:
+      name: production
+      url: ${{ steps.show-url.outputs.url }}
+
     permissions:
       contents: read
       id-token: write  # Required for Workload Identity Federation
@@ -104,6 +124,7 @@ jobs:
             --project $PROJECT_ID \
             --allow-unauthenticated \
             --env-vars-file=env-cloudrun.yaml \
+            --set-env-vars="GITHUB_APP_ID=${{ secrets.GITHUB_APP_ID }},INSTALLATION_ID=${{ secrets.INSTALLATION_ID }}" \
             --max-instances=10 \
             --cpu=1 \
             --memory=512Mi \
@@ -113,10 +134,11 @@ jobs:
             --platform=managed
 
       - name: Show deployment URL
+        id: show-url
         run: |
           URL=$(gcloud run services describe $SERVICE_NAME \
             --region $REGION \
             --project $PROJECT_ID \
             --format='value(status.url)')
-          echo "🚀 Deployed to: $URL"
-
+          echo "url=$URL" >> $GITHUB_OUTPUT
+          echo "Deployed to: $URL"

.gitignore

@@ -2,6 +2,9 @@
 github-copier
 code-copier
 copier
+config-validator
+test-webhook
+test-pem
 *.exe
 *.exe~
 *.dll

.gitleaksignore

@@ -0,0 +1,2 @@
+# Example placeholder string in .env.local.example (not a real key)
+configs/.env.local.example:private-key:77

.pre-commit-config.yaml

@@ -5,11 +5,16 @@ repos:
     hooks:
       - id: gitleaks
 
-  # Go linting
-  - repo: https://github.com/golangci/golangci-lint
-    rev: v1.62.2
+  # Go linting - requires golangci-lint v2 installed locally:
+  #   go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.9.0
+  - repo: local
     hooks:
       - id: golangci-lint
+        name: golangci-lint
+        entry: golangci-lint run --fix
+        language: system
+        pass_filenames: false
+        types: [go]
 
   # Local Go hooks
   - repo: local

AGENT.md

@@ -5,23 +5,42 @@ Webhook service: PR merged → match files → transform paths → copy to targe
 ## File Map
 
 ```
-app.go                              # entrypoint, HTTP server
+app.go                              # entrypoint, HTTP server, graceful shutdown
 services/
-  webhook_handler_new.go            # HandleWebhookWithContainer()
-  workflow_processor.go             # ProcessWorkflow() - core logic
+  webhook_handler_new.go            # HandleWebhookWithContainer() orchestrator
+  workflow_processor.go             # ProcessWorkflow() - core file matching logic
   pattern_matcher.go                # MatchFile(pattern, path) bool
-  github_auth.go                    # ConfigurePermissions() error
-  github_read.go                    # GetFilesChangedInPr(), RetrieveFileContents()
-  github_write_to_target.go         # AddFilesToTargetRepoBranch()
-  github_write_to_source.go         # UpdateDeprecationFile()
-  file_state_service.go             # tracks upload/deprecate queues
+  token_manager.go                  # TokenManager (thread-safe token state, sync.RWMutex)
+  github_auth.go                    # ConfigurePermissions(), JWT generation
+  github_read.go                    # GetFilesChangedInPr() (GraphQL), RetrieveFileContents()
+  github_write_to_target.go         # AddFilesToTargetRepos(), addFilesViaPR()
+  github_write_to_source.go         # UpdateDeprecationFile(filesToDeprecate)
+  rate_limit.go                     # RateLimitTransport (auto-retry on 403/429)
+  delivery_tracker.go               # DeliveryTracker (webhook idempotency via X-GitHub-Delivery)
+  errors.go                         # Sentinel errors (ErrRateLimited, ErrNotFound, etc.)
+  logger.go                         # slog JSON handler, LogCritical, LogAndReturnError
+  file_state_service.go             # Tracks upload/deprecate queues (thread-safe)
   main_config_loader.go             # LoadConfig() with $ref support
+  config_loader.go                  # Config loading & validation
   service_container.go              # DI container
+  health_metrics.go                 # /health (liveness), /ready (readiness), /metrics
+  audit_logger.go                   # MongoDB audit logging
+  slack_notifier.go                 # Slack notifications
+  pr_template_fetcher.go            # PR template resolution from target repos
 types/
   config.go                         # Workflow, Transformation, SourcePattern structs
   types.go                          # ChangedFile, UploadKey, UploadFileContent
 configs/environment.go              # Config struct, LoadEnvironment()
-tests/utils.go                      # test helpers, httpmock setup
+tests/utils.go                      # Test helpers, httpmock setup
+cmd/
+  config-validator/main.go          # CLI: validate configs, test patterns, init templates
+  test-webhook/main.go              # CLI: send test webhook payloads (with delivery ID)
+  test-pem/main.go                  # CLI: verify PEM key + App ID against GitHub API
+scripts/
+  ci-local.sh                       # Run full CI pipeline locally (build, test, lint, vet)
+  run-local.sh                      # Run app locally with dev settings
+  deploy-cloudrun.sh                # Deploy to Google Cloud Run
+  integration-test.sh               # End-to-end integration test
 ```
 
 ## Key Types
@@ -32,27 +51,31 @@ type PatternType string    // "prefix" | "glob" | "regex"
 type TransformationType string  // "move" | "copy" | "glob" | "regex"
 
 type Workflow struct {
-    Name           string
-    Source         SourceConfig      // Repo, Branch, Patterns []SourcePattern
-    Destination    DestinationConfig // Repo, Branch
-    Transformations []Transformation // Type, From, To, Pattern, Replacement
-    Commit         CommitConfig      // Strategy, Message, PRTitle, AutoMerge
+    Name             string
+    Source           Source              // Repo, Branch, InstallationID
+    Destination      Destination         // Repo, Branch
+    Transformations  []Transformation    // Type, From, To, Pattern, Replacement
+    Exclude          []string
+    CommitStrategy   *CommitStrategyConfig // Type (direct|pull_request), PRTitle, PRBody, AutoMerge
+    DeprecationCheck *DeprecationConfig
 }
 
 // types/types.go
 type ChangedFile struct { Path, Status string }  // Status: "ADDED"|"MODIFIED"|"DELETED"
 type UploadKey struct { RepoName, BranchPath string }
 ```
 
-## Global State (⚠️ mutable)
+## State Management
 
-```go
-// services/github_write_to_target.go
-var FilesToUpload map[UploadKey]UploadFileContent
-// services/github_auth.go
-var InstallationAccessToken string
-var OrgTokens map[string]string
-```
+All mutable state is encapsulated in `TokenManager` (thread-safe via `sync.RWMutex`):
+- Installation access token
+- Per-org installation tokens with expiry
+- Cached JWT
+- HTTP client
+
+Per-request file state is managed via `FileStateService` in the `ServiceContainer`.
+
+Webhook idempotency is handled by `DeliveryTracker` (TTL-based, in-memory).
 
 ## Config Example
 
@@ -62,14 +85,15 @@ workflows:
     source: { repo: "org/src", branch: "main", patterns: [{type: glob, pattern: "docs/**"}] }
     destination: { repo: "org/dest", branch: "main" }
     transformations: [{ type: move, from: "docs/", to: "public/" }]
-    commit: { strategy: pr, message: "Sync" }  # strategy: direct|pr
+    commit_strategy: { type: pull_request, pr_title: "Sync docs" }  # type: direct|pull_request
 ```
 
 ## Test Commands
 
 ```bash
-go test ./...                                    # all
+go test -race ./...                              # all with race detector
 go test ./services/... -run TestWorkflow -v      # specific
+golangci-lint run ./...                          # lint
 ```
 
 ## Edit Patterns
@@ -80,11 +104,17 @@ go test ./services/... -run TestWorkflow -v      # specific
 | New pattern type | `types/config.go` (PatternType) → `pattern_matcher.go` |
 | New config field | `types/config.go` (struct) → consumers in `workflow_processor.go` |
 | Webhook logic | `webhook_handler_new.go` |
+| Rate limit behavior | `rate_limit.go` |
+| Auth flow | `github_auth.go` + `token_manager.go` |
+| CLI tool | `cmd/<tool>/main.go` + `cmd/<tool>/README.md` |
 
 ## Conventions
 
 - Return `error`, never `log.Fatal`
 - Wrap errors: `fmt.Errorf("context: %w", err)`
+- Use sentinel errors from `errors.go` where appropriate
 - Nil-check GitHub API responses before dereference
+- Use `log/slog` for all logging (never `log` or `fmt.Print` for operational output)
 - Tests use `httpmock`; see `tests/utils.go`
+- Always run tests with `-race` flag
 - **Changelog**: Update `CHANGELOG.md` for all notable changes (follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/))

CHANGELOG.md

@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file.
 
+## Feb 2026
+
+### Added
+- **Rate limit handling** - Custom `RateLimitTransport` (`services/rate_limit.go`) automatically detects 403/429 responses with `X-RateLimit-Remaining` / `Retry-After` headers and retries with backoff
+- **Webhook idempotency** - `DeliveryTracker` (`services/delivery_tracker.go`) deduplicates webhook deliveries using `X-GitHub-Delivery` header with TTL-based cleanup
+- **Structured logging** - Migrated from `log` to `log/slog` with JSON handler (`services/logger.go`); compatible with Google Cloud Logging severity levels including custom `CRITICAL`
+- **Sentinel errors** - Centralized error definitions in `services/errors.go` (`ErrRateLimited`, `ErrNotFound`, etc.)
+- **Token manager** - Thread-safe `TokenManager` (`services/token_manager.go`) with `sync.RWMutex` for concurrent token access
+- **Readiness endpoint** - `/ready` probe checks GitHub auth, rate limit headroom, and MongoDB connectivity
+- **GitHub App manifest** - `github-app-manifest.yml` documenting required permissions and events
+- **Integration tests** - End-to-end tests for webhook processing pipeline (`services/integration_test.go`), GitHub auth flow, and extracted helper functions
+- **CI/CD improvements** - Added gosec security scanning, Trivy vulnerability scanning, GitHub Environment deployment gates
+
+### Changed
+- **Go version** - Upgraded from 1.24.0 to 1.26.0
+- **go-github** - Upgraded from v48 to v82 (`github.String` → `github.Ptr`, updated API signatures)
+- **mongo-driver** - Upgraded from v1 to v2 (updated import paths, API changes)
+- **Error handling** - All errors wrapped with `%w` for `errors.Is()`/`errors.As()` compatibility; removed bare `log.Fatal` calls
+- **Function decomposition** - Broke up `handleMergedPRWithContainer` (→ 5 helpers) and `addFilesViaPR` (→ 3 helpers) for readability
+- **Dot imports** - Removed all dot imports for explicit package references
+- **Health endpoint** - Split into `/health` (liveness) and `/ready` (readiness) with deep dependency checks
+- **HTTP method check** - Webhook handler rejects non-POST requests with 405
+- **MongoDB timeouts** - Configured explicit connection, server selection, and operation timeouts
+- **Dockerfile** - Pinned base image, added non-root user, added `HEALTHCHECK` instruction
+- **Environment config** - Moved org-specific values to deployment-time configuration; committed env file contains only non-secret defaults
+
 ## 17 Dec 2025
 
 ### Added

Dockerfile

@@ -1,5 +1,5 @@
 # Build stage
-FROM golang:1.24.0-alpine AS builder
+FROM golang:1.26.0-alpine AS builder
 
 # Install build dependencies
 RUN apk add --no-cache git ca-certificates
@@ -16,23 +16,32 @@ COPY . .
 # Build the binary
 # CGO_ENABLED=0 for static binary (no C dependencies)
 # -ldflags="-w -s" strips debug info to reduce size
-RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o examples-copier .
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o github-copier .
 
-# Runtime stage
-FROM alpine:latest
+# Runtime stage - pin to specific version for reproducible builds
+FROM alpine:3.21
 
 # Install ca-certificates for HTTPS requests
 RUN apk --no-cache add ca-certificates
 
-WORKDIR /root/
+# Create non-root user
+RUN addgroup -S appgroup && adduser -S appuser -G appgroup
+
+WORKDIR /home/appuser
 
 # Copy binary from builder
-COPY --from=builder /app/examples-copier .
+COPY --from=builder /app/github-copier .
+
+# Switch to non-root user
+USER appuser
 
 # Cloud Run sets PORT environment variable
 # Our app reads it from config.Port (defaults to 8080)
 EXPOSE 8080
 
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1
+
 # Run the binary
-CMD ["./examples-copier"]
+CMD ["./github-copier"]