diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index c4fb262..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,223 +0,0 @@ -name: CI/CD Pipeline - -on: - push: - branches: [main, develop] - pull_request: - branches: [main, develop] - -env: - NODE_VERSION: '18' - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - test: - name: Test and Quality Checks - runs-on: ubuntu-latest - - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: propchain_test - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run ESLint - run: npm run lint -- --max-warnings 0 - - - name: Check Prettier formatting - run: npm run format -- --check --ignore-path .gitignore - - - name: Run TypeScript compilation - run: npm run build - - - name: Run unit tests - run: npm run test:unit - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/propchain_test - REDIS_HOST: localhost - REDIS_PORT: 6379 - NODE_ENV: test - JWT_SECRET: test-secret-key - ENCRYPTION_KEY: test-encryption-key-32-chars-long - API_KEY_RATE_LIMIT_PER_MINUTE: 60 - - - name: Run integration tests - run: npm run test:integration - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/propchain_test - REDIS_HOST: localhost - REDIS_PORT: 6379 - NODE_ENV: test - JWT_SECRET: test-secret-key - ENCRYPTION_KEY: test-encryption-key-32-chars-long - API_KEY_RATE_LIMIT_PER_MINUTE: 60 - - - name: Generate test coverage - run: npm run test:cov - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/propchain_test - REDIS_HOST: localhost - REDIS_PORT: 6379 - NODE_ENV: test - JWT_SECRET: test-secret-key - ENCRYPTION_KEY: test-encryption-key-32-chars-long - API_KEY_RATE_LIMIT_PER_MINUTE: 60 - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage/lcov.info - flags: unittests - name: codecov-umbrella - - security: - name: Security Scan - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - scan-ref: '.' - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - ignore-unfixed: true - continue-on-error: true - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' - continue-on-error: true - - - name: Run npm audit - run: npm audit --audit-level=moderate - continue-on-error: true - - build: - name: Build Docker Image - runs-on: ubuntu-latest - needs: [test, security] - if: github.event_name == 'push' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=sha,prefix={{branch}}- - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push Docker image - id: build - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - deploy-staging: - name: Deploy to Staging - runs-on: ubuntu-latest - needs: build - if: github.ref == 'refs/heads/develop' - environment: staging - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Deploy to staging - run: echo "Deploying to staging..." || true - - deploy-production: - name: Deploy to Production - runs-on: ubuntu-latest - needs: build - if: github.ref == 'refs/heads/main' - environment: production - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Deploy to production - run: echo "Deploying to production..." || true - - notify: - name: Notify on Failure - runs-on: ubuntu-latest - needs: [test, security] - if: always() - steps: - - name: Check for failures - run: | - if [ "${{ needs.test.result }}" = "failure" ] || [ "${{ needs.security.result }}" = "failure" ]; then - echo "PIPELINE_FAILED=true" >> $GITHUB_ENV - else - echo "PIPELINE_FAILED=false" >> $GITHUB_ENV - echo "All checks passed." - fi - - - name: Notify Slack - if: env.PIPELINE_FAILED == 'true' - uses: 8398a7/action-slack@v3 - with: - status: failure - channel: '#ci-cd' - text: 'Pipeline failed for ${{ github.repository }}' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - continue-on-error: true diff --git a/.github/workflows/comprehensive-testing.yml b/.github/workflows/comprehensive-testing.yml deleted file mode 100644 index 0a44d65..0000000 --- a/.github/workflows/comprehensive-testing.yml +++ /dev/null @@ -1,484 +0,0 @@ -name: Comprehensive Testing Pipeline - -on: - push: - branches: [main, develop, 'feature/*'] - pull_request: - branches: [main, develop] - -env: - NODE_VERSION: '18' - DATABASE_URL: 'postgresql://test:test@localhost:5432/propchain_test' - REDIS_URL: 'redis://localhost:6379/1' - -jobs: - # Code Quality and Linting - quality: - name: Code Quality Checks - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run ESLint - run: npm run lint -- --max-warnings 0 - - - name: Run Prettier check - run: npm run format -- --check - - - name: TypeScript compilation check - run: npm run build - - # Unit Tests with Coverage - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - needs: quality - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run unit tests with coverage - run: npm run test:unit -- --coverage --ci - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage/lcov.info - flags: unit-tests - name: unit-test-coverage - - - name: Check coverage thresholds - run: | - COVERAGE=$(npm run test:coverage:badge) - if [ "$COVERAGE" -lt 80 ]; then - echo "Coverage $COVERAGE% is below 80% threshold" - exit 1 - fi - - # Integration Tests - integration-tests: - name: Integration Tests - runs-on: ubuntu-latest - needs: quality - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: test - POSTGRES_USER: test - POSTGRES_DB: propchain_integration - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Wait for services - run: | - timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done' - timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done' - - - name: Run database migrations - run: | - npm run db:generate - npm run migrate:deploy - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_integration - - - name: Run integration tests - run: npm run test:integration - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_integration - REDIS_URL: redis://localhost:6379/2 - - # End-to-End Tests - e2e-tests: - name: End-to-End Tests - runs-on: ubuntu-latest - needs: [unit-tests, integration-tests] - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: test - POSTGRES_USER: test - POSTGRES_DB: propchain_e2e - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Wait for services - run: | - timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done' - timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done' - - - name: Run database migrations - run: | - npm run db:generate - npm run migrate:deploy - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_e2e - - - name: Build application - run: npm run build - - - name: Start application - run: npm start & - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_e2e - REDIS_URL: redis://localhost:6379/3 - NODE_ENV: production - - - name: Wait for application - run: timeout 60 bash -c 'until curl -f http://localhost:3000/health; do sleep 2; done' - - - name: Run E2E tests - run: npm run test:e2e - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_e2e - REDIS_URL: redis://localhost:6379/3 - - # Performance Tests - performance-tests: - name: Performance Tests - runs-on: ubuntu-latest - needs: unit-tests - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: test - POSTGRES_USER: test - POSTGRES_DB: propchain_performance - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Wait for services - run: | - timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done' - timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done' - - - name: Run database migrations - run: | - npm run db:generate - npm run migrate:deploy - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_performance - - - name: Run performance tests - run: npm run test:performance - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_performance - REDIS_URL: redis://localhost:6379/4 - - - name: Upload performance results - uses: actions/upload-artifact@v3 - with: - name: performance-results - path: test-results/performance/ - - # Security Tests - security-tests: - name: Security Tests - runs-on: ubuntu-latest - needs: unit-tests - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run security tests - run: npm run test:security - - - name: Run npm audit - run: npm audit --audit-level=moderate - continue-on-error: true - - - name: Run Snyk security scan - run: | - npx snyk test --severity-threshold=high - continue-on-error: true - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - - name: Run CodeQL Analysis - uses: github/codeql-action/init@v2 - with: - languages: javascript - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - - # Load Testing - load-tests: - name: Load Testing - runs-on: ubuntu-latest - needs: integration-tests - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: test - POSTGRES_USER: test - POSTGRES_DB: propchain_load - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Install k6 - run: | - sudo gpg -k - sudo gpg --no-default-keyring --keyring /usr/share/keyrings/debian-archive-keyring.gpg --import <(curl -sSL 'https://dl.k6.io/key.gpg') - echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list - sudo apt-get update - sudo apt-get install k6 - - - name: Wait for services - run: | - timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done' - timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done' - - - name: Run database migrations - run: | - npm run db:generate - npm run migrate:deploy - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_load - - - name: Build application - run: npm run build - - - name: Start application - run: npm start & - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_load - REDIS_URL: redis://localhost:6379/5 - NODE_ENV: production - - - name: Wait for application - run: timeout 60 bash -c 'until curl -f http://localhost:3000/health; do sleep 2; done' - - - name: Run load tests - run: npm run test:load - env: - DATABASE_URL: postgresql://test:test@localhost:5432/propchain_load - REDIS_URL: redis://localhost:6379/5 - - - name: Upload load test results - uses: actions/upload-artifact@v3 - with: - name: load-test-results - path: test-results/load/ - - # Build and Deploy - build: - name: Build and Deploy - runs-on: ubuntu-latest - needs: [unit-tests, integration-tests, e2e-tests, security-tests] - if: github.ref == 'refs/heads/main' - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Build application - run: npm run build - - - name: Create deployment artifact - run: | - tar -czf deployment.tar.gz dist/ package.json package-lock.json - - - name: Upload deployment artifact - uses: actions/upload-artifact@v3 - with: - name: deployment-artifact - path: deployment.tar.gz - - # Test Results Summary - test-summary: - name: Test Summary - runs-on: ubuntu-latest - needs: [unit-tests, integration-tests, e2e-tests, performance-tests, security-tests, load-tests] - if: always() - steps: - - name: Download all artifacts - uses: actions/download-artifact@v3 - - - name: Generate test summary - run: | - echo "# Test Results Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Test Type | Status | Details |" >> $GITHUB_STEP_SUMMARY - echo "|-----------|--------|---------|" >> $GITHUB_STEP_SUMMARY - echo "| Unit Tests | ${{ needs.unit-tests.result }} | Coverage: $(npm run test:coverage:badge 2>/dev/null || echo 'N/A')% |" >> $GITHUB_STEP_SUMMARY - echo "| Integration Tests | ${{ needs.integration-tests.result }} | Database integration |" >> $GITHUB_STEP_SUMMARY - echo "| E2E Tests | ${{ needs.e2e-tests.result }} | Full application flow |" >> $GITHUB_STEP_SUMMARY - echo "| Performance Tests | ${{ needs.performance-tests.result || 'skipped' }} | Load and performance |" >> $GITHUB_STEP_SUMMARY - echo "| Security Tests | ${{ needs.security-tests.result }} | Security scanning |" >> $GITHUB_STEP_SUMMARY - echo "| Load Tests | ${{ needs.load-tests.result || 'skipped' }} | Stress testing |" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Coverage Report" >> $GITHUB_STEP_SUMMARY - echo "- [View detailed coverage report](https://codecov.io/gh/${{ github.repository })" >> $GITHUB_STEP_SUMMARY - - # Notification - notify: - name: Notify Results - runs-on: ubuntu-latest - needs: [unit-tests, integration-tests, e2e-tests, security-tests] - if: always() - steps: - - name: Notify on success - if: needs.unit-tests.result == 'success' && needs.integration-tests.result == 'success' && needs.e2e-tests.result == 'success' && needs.security-tests.result == 'success' - run: | - echo "✅ All tests passed successfully!" - echo "Ready for deployment." - - - name: Notify on failure - if: needs.unit-tests.result == 'failure' || needs.integration-tests.result == 'failure' || needs.e2e-tests.result == 'failure' || needs.security-tests.result == 'failure' - run: | - echo "❌ Some tests failed!" - echo "Please check the logs and fix the issues before deployment." diff --git a/.gitignore b/.gitignore index 62f60dd..21a1569 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ out/ .env.development.local .env.test.local .env.production.local +.env.example +src/backup-recovery/.env.example # IDE and editor files .vscode/ diff --git a/SETUP.md b/SETUP.md deleted file mode 100644 index 7ed483d..0000000 --- a/SETUP.md +++ /dev/null @@ -1,341 +0,0 @@ -# PropChain Backend Setup Guide - -This guide will help you set up the PropChain Backend development environment from scratch. - -## Prerequisites - -Before you begin, ensure you have the following installed: - -- **Node.js** v18+ (LTS recommended) -- **npm** or **yarn** package manager -- **PostgreSQL** v14+ -- **Redis** v6+ -- **Git** version control -- **Docker** & **Docker Compose** (optional, for containerized setup) - -## Quick Start - -### 1. Clone the Repository - -```bash -git clone https://github.com/MettaChain/PropChain-BackEnd.git -cd PropChain-BackEnd -``` - -### 2. Install Dependencies - -```bash -# Using npm -npm install - -# Or using yarn -yarn install -``` - -### 3. Environment Configuration - -```bash -# Copy the environment template -cp .env.example .env - -# Edit the .env file with your configuration -nano .env -``` - -**Required Environment Variables:** - -- `DATABASE_URL` - PostgreSQL connection string -- `JWT_SECRET` - Secret for JWT token signing -- `ENCRYPTION_KEY` - 32-character encryption key -- `RPC_URL` - Blockchain RPC endpoint - -### 4. Database Setup - -```bash -# Create database -createdb propchain - -# Run database migrations -npm run migrate - -# Generate Prisma client -npm run db:generate - -# (Optional) Seed with test data -npm run db:seed -``` - -### 5. Start Development Server - -```bash -# Start in development mode with hot reload -npm run start:dev - -# Or with debug logging -npm run start:debug -``` - -The API will be available at `http://localhost:3000` with interactive Swagger docs at `http://localhost:3000/api/docs`. - -## Docker Setup (Recommended) - -### Using Docker Compose - -```bash -# Start all services (database, redis, api) -docker-compose up -d - -# View logs -docker-compose logs -f api - -# Stop services -docker-compose down -``` - -### Individual Services - -```bash -# Start only database -docker-compose up -d postgres redis - -# Start API with local development -npm run start:dev -``` - -## Development Workflow - -### Code Quality - -```bash -# Run linting -npm run lint - -# Fix linting issues -npm run lint -- --fix - -# Format code -npm run format - -# Type checking -npm run build -``` - -### Testing - -```bash -# Run all tests -npm test - -# Run specific test suites -npm run test:unit -npm run test:integration -npm run test:e2e - -# Generate coverage report -npm run test:cov - -# Run tests in watch mode -npm run test:watch -``` - -### Database Operations - -```bash -# Create new migration -npx prisma migrate dev --name migration_name - -# Reset database -npm run db:reset - -# View database in browser -npm run db:studio - -# Generate Prisma client -npm run db:generate -``` - -## Project Structure - -``` -src/ -├── common/ # Shared utilities and middleware -│ ├── filters/ # Exception filters -│ ├── interceptors/ # Response interceptors -│ ├── logger/ # Winston logging -│ └── decorators/ # Custom decorators -├── config/ # Configuration management -├── database/ # Database models and services -├── health/ # Health check endpoints -├── modules/ # Business logic modules -│ ├── auth/ # Authentication -│ ├── users/ # User management -│ ├── properties/ # Property management -│ ├── transactions/ # Transaction handling -│ └── blockchain/ # Blockchain integration -├── app.module.ts # Root module -└── main.ts # Application entry point -``` - -## Environment Variables - -### Application Configuration - -- `NODE_ENV` - Environment (development/staging/production) -- `PORT` - Server port (default: 3000) -- `API_PREFIX` - API route prefix (default: api) -- `CORS_ORIGIN` - Allowed CORS origins - -### Database - -- `DATABASE_URL` - PostgreSQL connection string - -### Redis - -- `REDIS_HOST` - Redis server host -- `REDIS_PORT` - Redis server port -- `REDIS_PASSWORD` - Redis password (if required) - -### Security - -- `JWT_SECRET` - JWT signing secret -- `JWT_EXPIRES_IN` - JWT token expiration -- `ENCRYPTION_KEY` - Data encryption key - -### Blockchain - -- `BLOCKCHAIN_NETWORK` - Blockchain network (sepolia/mainnet) -- `RPC_URL` - Blockchain RPC endpoint -- `PRIVATE_KEY` - Private key for transactions (development only) - -## API Documentation - -Once the server is running, visit: - -- **Swagger UI**: `http://localhost:3000/api/docs` -- **Health Check**: `http://localhost:3000/api/health` -- **Configuration**: `http://localhost:3000/api/configuration` - -## Troubleshooting - -### Common Issues - -**Database Connection Error** - -```bash -# Check PostgreSQL status -pg_ctl status - -# Reset database -npm run db:reset -``` - -**Redis Connection Error** - -```bash -# Check Redis status -redis-cli ping - -# Restart Redis -docker-compose restart redis -``` - -**Module Not Found Errors** - -```bash -# Clear node modules and reinstall -rm -rf node_modules package-lock.json -npm install -``` - -**Port Already in Use** - -```bash -# Find process using port 3000 -lsof -ti:3000 - -# Kill process -kill -9 $(lsof -ti:3000) -``` - -### Getting Help - -- Check the [GitHub Issues](https://github.com/MettaChain/PropChain-BackEnd/issues) -- Review the [API Documentation](http://localhost:3000/api/docs) -- Join our [Discord Community](https://discord.gg/propchain) - -## Contributing - -1. Fork the repository -2. Create a feature branch: `git checkout -b feature/amazing-feature` -3. Commit your changes: `git commit -m 'Add amazing feature'` -4. Push to the branch: `git push origin feature/amazing-feature` -5. Open a Pull Request - -Please read our [Contributing Guide](./CONTRIBUTING.md) for detailed guidelines. - -## Production Deployment - -### Environment Setup - -1. Set `NODE_ENV=production` -2. Configure production database and Redis -3. Set secure JWT secrets and encryption keys -4. Configure proper CORS origins -5. Set up SSL certificates - -### Docker Deployment - -```bash -# Build production image -docker build -t propchain-backend . - -# Run with production configuration -docker run -d \ - --name propchain-api \ - -p 3000:3000 \ - --env-file .env.production \ - propchain-backend -``` - -### Kubernetes - -```bash -# Apply Kubernetes manifests -kubectl apply -f k8s/ - -# Check deployment status -kubectl get pods -l app=propchain-backend -``` - -## Security Considerations - -- Never commit `.env` files or secrets -- Use strong, unique JWT secrets -- Enable rate limiting in production -- Use HTTPS in production -- Regularly update dependencies -- Implement proper input validation -- Use environment-specific configurations - -## Performance Optimization - -- Enable Redis caching for frequently accessed data -- Use database connection pooling -- Implement proper indexing -- Monitor application metrics -- Use CDN for static assets -- Optimize database queries - -## Monitoring and Logging - -- Application logs are stored in `logs/` directory -- Use structured logging for better monitoring -- Set up external monitoring (Prometheus/Grafana) -- Configure error tracking (Sentry) -- Monitor database performance - ---- - -**Happy coding! 🚀** - -For additional support, reach out to the PropChain team at support@propchain.io diff --git a/src/backup-recovery/DISASTER_RECOVERY_RUNBOOK.md b/src/backup-recovery/DISASTER_RECOVERY_RUNBOOK.md new file mode 100644 index 0000000..f78bb57 --- /dev/null +++ b/src/backup-recovery/DISASTER_RECOVERY_RUNBOOK.md @@ -0,0 +1,589 @@ +# Backup and Disaster Recovery System - Operational Runbooks + +## Table of Contents + +1. [Database Backup Operations](#database-backup-operations) +2. [Document Backup Operations](#document-backup-operations) +3. [Disaster Recovery Procedures](#disaster-recovery-procedures) +4. [Emergency Recovery Steps](#emergency-recovery-steps) +5. [Monitoring and Alerting](#monitoring-and-alerting) +6. [Incident Response](#incident-response) + +--- + +## Database Backup Operations + +### 1. Manual Full Database Backup + +**Purpose**: Create a complete point-in-time snapshot of the production database + +**Procedure**: + +```bash +# Using API +curl -X POST http://localhost:3000/v1/backup-recovery/database/backup/full \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "tags": { + "type": "manual", + "reason": "pre-deployment", + "operator": "admin@propchain.local" + } + }' + +# Using CLI +npm run db:backup +``` + +**Expected Outcome**: +- HTTP Status: 202 (Accepted) +- Response includes backup ID, timestamp, and estimated duration +- Backup process runs asynchronously in background +- Monitor progress via `/backup-recovery/database/backups/:backupId` + +**Troubleshooting**: +- If backup times out (>30 minutes), check database locks: `SELECT * FROM pg_locks;` +- If storage space insufficient, run retention policies: `POST /backup-recovery/retention/enforce-policies` + +--- + +### 2. Verify Database Backup Integrity + +**Purpose**: Ensure backup is restorable and not corrupted + +**Procedure**: + +```bash +# Trigger verification +curl -X POST http://localhost:3000/v1/backup-recovery/verification/verify/:backupId \ + -H "Authorization: Bearer " + +# Get verification results +curl -X GET http://localhost:3000/v1/backup-recovery/verification/:backupId \ + -H "Authorization: Bearer " +``` + +**Expected Results**: +- Checksum verification: PASSED +- File accessibility: Accessible +- Restorability test: Restorable +- Duration: <5 minutes + +**If Verification Fails**: +1. Check backup file exists: `ls -lh backups/database/full/:backupId.dump` +2. Check file permissions: `stat backups/database/full/:backupId.dump` +3. Test database connectivity +4. Recreate backup if corrupted + +--- + +### 3. Point-in-Time Recovery (PITR) + +**Purpose**: Restore database to a specific point in time + +**Procedure**: + +```bash +# Step 1: Get PITR information for target timestamp +curl -X GET "http://localhost:3000/v1/backup-recovery/recovery/point-in-time/2024-01-15T14:30:00Z" \ + -H "Authorization: Bearer " + +# Step 2: Initiate PITR +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/point-in-time-recovery \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "targetTimestamp": "2024-01-15T14:30:00Z", + "backupId": "full_1705329600000_abc123", + "targetEnvironment": "staging" + }' + +# Step 3: Monitor recovery progress +curl -X GET http://localhost:3000/v1/backup-recovery/disaster-recovery/status \ + -H "Authorization: Bearer " +``` + +**Expected Duration**: 20-45 minutes (depending on database size) + +**Validation**: +- Check record counts match expected values +- Query known data points +- Verify application can connect +- Test critical business flows + +**Rollback**: +- If recovery fails, promote previous replica +- Issue: `POST /backup-recovery/disaster-recovery/failover` to alternate region + +--- + +### 4. Backup Retention Policy + +**Policy**: +- Daily full backups: Retain 30 days +- Incremental backups: Retain 7 days +- Weekly point-in-time: Retain 12 weeks +- Monthly snapshots: Retain 12 months +- Archived backups: Retain 7 years (cold storage) + +**Automatic Enforcement**: +- Runs daily at 01:00 UTC +- Old backups automatically deleted +- Expired backups moved to archive storage + +**Manual Enforcement**: + +```bash +curl -X POST http://localhost:3000/v1/backup-recovery/retention/enforce-policies \ + -H "Authorization: Bearer " +``` + +--- + +## Document Backup Operations + +### 1. Manual Document Backup + +**Purpose**: Backup all user-uploaded documents + +**Procedure**: + +```bash +curl -X POST http://localhost:3000/v1/backup-recovery/documents/backup \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "tags": { + "type": "manual", + "reason": "audit" + } + }' +``` + +**Locations Replicated To**: +- Local storage: `backups/documents/snapshots/` +- Azure Blob Storage: `document-backups/2024/01/` +- AWS S3: `s3://propchain-backups/document-backups/2024/01/` + +--- + +### 2. Verify Document Backup + +**Purpose**: Ensure all documents are properly backed up + +**Procedure**: + +```bash +# Trigger verification +curl -X POST http://localhost:3000/v1/backup-recovery/documents/backups/:backupId/verify \ + -H "Authorization: Bearer " + +# Get statistics +curl -X GET http://localhost:3000/v1/backup-recovery/documents/statistics \ + -H "Authorization: Bearer " +``` + +**Verification Checks**: +- Archive integrity +- Manifest validity +- File checksums +- All data accessible + +--- + +### 3. Restore Document from Backup + +**Procedure**: + +```bash +# 1. Extract backup archive +tar -xzf backups/documents/snapshots/docs_1705329600000_abc123.tar.gz + +# 2. Verify manifest +cat backups/documents/snapshots/docs_1705329600000_abc123/MANIFEST.json + +# 3. Restore specific document +cp -r docs_backup/document_id/* uploads/documents/document_id/ + +# 4. Verify restoration +curl -X GET http://localhost:3000/v1/documents/document_id \ + -H "Authorization: Bearer " +``` + +--- + +## Disaster Recovery Procedures + +### 1. Create Disaster Recovery Plan + +**Purpose**: Define recovery strategy for critical failures + +**Procedure**: + +```bash +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/plans \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "id": "production_dr_plan", + "name": "Production Disaster Recovery Plan", + "rpo": "1h", + "rto": "4h", + "failoverRegions": ["us-east-1", "eu-west-1"], + "healthCheckInterval": 300000, + "automaticFailover": false, + "notificationChannels": ["slack", "email"], + "testInterval": 604800000 + }' +``` + +**Parameters**: +- **RPO** (Recovery Point Objective): Maximum acceptable data loss (1 hour = max 1 hour of data loss) +- **RTO** (Recovery Time Objective): Maximum acceptable downtime (4 hours) +- **failoverRegions**: Ordered list of fallback regions +- **automaticFailover**: Enable automated failover (false = manual only) + +--- + +### 2. Automated DR Testing + +**Purpose**: Validate recovery procedures without impacting production + +**Procedure**: + +```bash +# Trigger manual DR test +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/test/production_dr_plan \ + -H "Authorization: Bearer " + +# Monitor test progress +curl -X GET http://localhost:3000/v1/backup-recovery/disaster-recovery/status \ + -H "Authorization: Bearer " +``` + +**Test Steps**: +1. Create isolated test environment +2. Restore latest production backup +3. Run application smoke tests +4. Validate data consistency +5. Cleanup test environment + +**Expected Duration**: 30-60 minutes + +**Review Results**: +```bash +curl -X GET http://localhost:3000/v1/backup-recovery/verification/lifecycle-stats \ + -H "Authorization: Bearer " +``` + +--- + +### 3. Manual Failover to Alternate Region + +**Purpose**: Redirect traffic to standby region during primary region failure + +**Prerequisites**: +- Verify DR plan exists and is active +- Confirm target region has current replica +- Notify stakeholders + +**Procedure**: + +```bash +# Step 1: Initiate failover +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/failover \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "planId": "production_dr_plan", + "targetRegion": "us-east-1" + }' + +# Response: +# { +# "status": "INITIATED", +# "failoverStartTime": "2024-01-15T10:00:00Z", +# "estimatedCompletionTime": "2024-01-15T10:30:00Z" +# } + +# Step 2: Monitor failover progress +curl -X GET http://localhost:3000/v1/backup-recovery/disaster-recovery/status \ + -H "Authorization: Bearer " +``` + +**Failover Sequence**: +1. ✅ Notify stakeholders via all channels +2. ✅ Prepare target infrastructure +3. ✅ Promote read replica to primary +4. ✅ Update DNS records (TTL: 60 seconds) +5. ✅ Validate application connectivity +6. ✅ Monitor health checks +7. ✅ Confirm failover success + +**Expected Duration**: 10-15 minutes + +**Post-Failover**: +- Monitor application metrics for 30 minutes +- Verify all integrations are working +- Update status page +- Begin primary region investigation + +--- + +## Emergency Recovery Steps + +### Scenario 1: Complete Database Failure (Primary Region) + +**Symptoms**: +- Database connection errors from all applications +- Health checks failing +- No backups in primary region + +**Recovery Steps**: + +```bash +# Step 1: Assess situation +curl -X GET http://localhost:3000/v1/backup-recovery/disaster-recovery/status + +# Step 2: Verify backup availability in alternate regions +curl -X GET http://localhost:3000/v1/backup-recovery/database/backups?status=VERIFIED + +# Step 3: Initiate failover +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/failover \ + -H "Authorization: Bearer " \ + -d '{"planId": "production_dr_plan", "targetRegion": "eu-west-1"}' + +# Step 4: Monitor restoration +curl -X GET http://localhost:3000/v1/backup-recovery/disaster-recovery/status + +# Step 5: Investigate primary region +# - Check RDS status in AWS console +# - Review CloudWatch logs +# - Check storage capacity +# - Verify network connectivity +``` + +**RTO: 15-30 minutes** + +--- + +### Scenario 2: Data Corruption or Ransomware + +**Symptoms**: +- Unusual data modifications +- Large-scale deletions +- Unknown encryption applied + +**Recovery Steps**: + +```bash +# Step 1: IMMEDIATELY isolate affected systems +# - Stop replication +# - Block database access except for recovery team +# - Capture forensics from affected nodes + +# Step 2: Find last known-good backup +curl -X GET http://localhost:3000/v1/backup-recovery/recovery/point-in-time/$(date -d '1 hour ago' --rfc-3339=seconds) \ + -H "Authorization: Bearer " + +# Step 3: Perform point-in-time recovery to staging +curl -X POST http://localhost:3000/v1/backup-recovery/disaster-recovery/point-in-time-recovery \ + -H "Authorization: Bearer " \ + -d '{ + "targetTimestamp": "2024-01-15T09:00:00Z", + "backupId": "full_1705329600000_abc123", + "targetEnvironment": "forensics" + }' + +# Step 4: Validate recovered data +# - Sample data queries +# - Application connectivity +# - No corruption indicators + +# Step 5: If validated, promote to production +# - DNS update +# - Monitor +# - Communication with users + +# Step 6: Forensic analysis +# - Review audit logs +# - Identify attack vector +# - Update security policies +``` + +**RTO: 1-2 hours** + +--- + +### Scenario 3: Document Storage Failure + +**Symptoms**: +- Document download failures +- Storage replication errors +- High latency + +**Recovery Steps**: + +```bash +# Step 1: Check backup status +curl -X GET http://localhost:3000/v1/backup-recovery/documents/statistics + +# Step 2: Identify affected documents +curl -X GET http://localhost:3000/v1/documents?status=failed + +# Step 3: Restore from latest backup +# Extract backup archive and restore missing documents +tar -xzf backups/documents/snapshots/docs_latest.tar.gz +cp -r docs_backup/* uploads/documents/ + +# Step 4: Verify restoration +curl -X GET http://localhost:3000/v1/backup-recovery/documents/backups/:backupId/verify + +# Step 5: Resume normal operations +``` + +**RTO: 30 minutes** + +--- + +## Monitoring and Alerting + +### 1. Critical Alerts + +**Alert**: Backup Failed (CRITICAL) +- **Action**: Immediate investigation required +- **Steps**: + ```bash + curl -X GET http://localhost:3000/v1/backup-recovery/monitoring/alerts?severity=CRITICAL + ``` + +**Alert**: Backup Timeout (CRITICAL) +- **Action**: Kill backup process, investigate database locks, restart backup + +**Alert**: Storage Full (CRITICAL) +- **Action**: Run retention policies immediately or expand storage + +--- + +### 2. Monitoring Dashboard + +Access operational metrics: + +```bash +curl -X GET http://localhost:3000/v1/backup-recovery/monitoring/dashboard \ + -H "Authorization: Bearer " + +# Response includes: +# { +# "alerts": { "critical": 0, "high": 1, "total": 2 }, +# "recentBackups": [...] +# "systemHealth": { "status": "HEALTHY" } +# } +``` + +--- + +### 3. Alert Management + +```bash +# Acknowledge alert +curl -X POST http://localhost:3000/v1/backup-recovery/monitoring/alerts/:alertId/acknowledge \ + -H "Authorization: Bearer " \ + -d '{"acknowledgedBy": "admin@propchain.local"}' + +# Resolve alert +curl -X POST http://localhost:3000/v1/backup-recovery/monitoring/alerts/:alertId/resolve \ + -H "Authorization: Bearer " +``` + +--- + +## Incident Response + +### Incident Severity Levels + +| Level | Response Time | Escalation | +|-------|--------------|------------| +| P1 - Critical | 15 minutes | VP Engineering + DevOps | +| P2 - High | 30 minutes | Engineering Lead + DevOps | +| P3 - Medium | 1 hour | DevOps Team | +| P4 - Low | 4 hours | DevOps Team | + +--- + +### Incident Communication Template + +``` +Subject: [INCIDENT] Backup System - + +SEVERITY: P<1-4> +START_TIME: +IMPACT: +ESTIMATED_RECOVERY: