diff --git a/.circleci/config.yml b/.circleci/config.yml index 88a98ff9..b5d51bf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,121 +2,241 @@ version: 2.1 orbs: aws-cli: circleci/aws-cli@3.1 - aws-ecr: circleci/aws-ecr@8.2.1 - aws-ecs: circleci/aws-ecs@3.2.0 + aws-ecr: circleci/aws-ecr@9.4.0 + aws-eks: circleci/aws-eks@2.2.0 + kubernetes: circleci/kubernetes@1.3 parameters: aws-region: type: string default: "us-east-1" - ecr-repository: + aws-role-arn: type: string - default: "transformers-mage" - ecs-cluster: + default: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/CircleCI-OIDC-Role" + ecr-repository-prefix: type: string - default: "synthetix-data" - ecs-service: + default: "synthetix-data/indexer" + eks-cluster-name: type: string - default: "transformers-mage-service" + default: "snx" jobs: - build-and-push: + build-and-push-images: docker: - - image: cimg/python:3.10 + - image: cimg/base:2023.03 + parameters: + network: + type: string steps: - checkout - setup_remote_docker: - version: 20.10.14 docker_layer_caching: true - aws-cli/setup: - role-arn: ${AWS_ROLE_ARN} - aws-region: << pipeline.parameters.aws-region >> + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION - # Build and push Docker image - - aws-ecr/build-and-push-image: - aws-access-key-id: ${AWS_ACCESS_KEY_ID} - aws-secret-access-key: ${AWS_SECRET_ACCESS_KEY} - account-url: ${AWS_ECR_ACCOUNT_URL} - repo: << pipeline.parameters.ecr-repository >> - region: << pipeline.parameters.aws-region >> - tag: "${CIRCLE_SHA1},latest" - path: ./transformers-mage - dockerfile: ./transformers-mage/Dockerfile + # Build and push Docker image for specific network + - aws-ecr/build_and_push_image: + account_id: ${AWS_ACCOUNT_ID} + auth: + - aws-cli/setup: + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION + region: ${AWS_REGION} + repo: mage-synthetix-prod-<< parameters.network >> + tag: "latest" + build_path: ./indexers/<< parameters.network >> + path: ./indexers/<< parameters.network >> + dockerfile: Dockerfile - deploy-to-ecs: + deploy-to-eks: docker: - image: cimg/python:3.10 steps: - checkout - aws-cli/setup: - role-arn: ${AWS_ROLE_ARN} - aws-region: << pipeline.parameters.aws-region >> + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION + + - kubernetes/install-kubectl - # Update ECS task definition with new image + - aws-eks/update-kubeconfig-with-authenticator: + cluster-name: snx + aws-region: ${AWS_REGION} + + # Get Aurora DB endpoint from CloudFormation outputs - run: - name: Fetch current task definition + name: Get Aurora DB details command: | - aws ecs describe-task-definition \ - --task-definition << pipeline.parameters.ecs-service >> \ - --region << pipeline.parameters.aws-region >> \ - --output json > task-definition.json + # Get DB endpoint from CloudFormation output + DB_ENDPOINT=$(aws cloudformation describe-stacks --stack-name synthetix-aurora-stack --query "Stacks[0].Outputs[?OutputKey=='DBEndpoint'].OutputValue" --output text) + echo "DB_ENDPOINT=$DB_ENDPOINT" >> $BASH_ENV + + # Get DB secret ARN + DB_SECRET_ARN=$(aws cloudformation describe-stacks --stack-name synthetix-aurora-stack --query "Stacks[0].Outputs[?OutputKey=='DBSecretArn'].OutputValue" --output text) + echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $BASH_ENV + # Update ConfigMap with Aurora DB endpoint - run: - name: Update task definition with new image + name: Update ConfigMap with Aurora DB endpoint command: | - # Extract the container definitions - jq '.taskDefinition.containerDefinitions' task-definition.json > container-defs.json + # Create namespace if it doesn't exist + if ! kubectl get namespace synthetix-data > /dev/null 2>&1; then + kubectl create namespace synthetix-data + fi + + # Create or update ConfigMap + cat > db-config.yaml \<>:${CIRCLE_SHA1}" \ - '.[0].image = $IMAGE' container-defs.json > updated-container-defs.json + kubectl apply -f db-config.yaml + + # Configure External Secrets + - run: + name: Set up External Secrets for Aurora DB credentials + command: | + # Replace placeholder account ID in YAML files + ACCOUNT_ID=$(echo $AWS_ECR_ACCOUNT_URL | cut -d'.' -f1) + sed -i "s/ACCOUNT_ID/$ACCOUNT_ID/g" k8s/external-secrets.yaml - # Get other task definition parameters - FAMILY=$(jq -r '.taskDefinition.family' task-definition.json) - EXECUTION_ROLE=$(jq -r '.taskDefinition.executionRoleArn' task-definition.json) - TASK_ROLE=$(jq -r '.taskDefinition.taskRoleArn' task-definition.json) - NETWORK_MODE=$(jq -r '.taskDefinition.networkMode' task-definition.json) - VOLUMES=$(jq '.taskDefinition.volumes' task-definition.json) - PLACEMENT_CONSTRAINTS=$(jq '.taskDefinition.placementConstraints' task-definition.json) + kubectl apply -f k8s/external-secrets.yaml - # Register new task definition - aws ecs register-task-definition \ - --family $FAMILY \ - --execution-role-arn $EXECUTION_ROLE \ - --task-role-arn $TASK_ROLE \ - --network-mode $NETWORK_MODE \ - --container-definitions file://updated-container-defs.json \ - --volumes "$VOLUMES" \ - --placement-constraints "$PLACEMENT_CONSTRAINTS" \ - --region << pipeline.parameters.aws-region >> \ - --output json > new-task-definition.json + # Create ExternalSecret to fetch Aurora DB credentials + cat > external-secret.yaml \<> $BASH_ENV + kubectl apply -f external-secret.yaml - # Update the ECS service with the new task definition - - aws-ecs/update-service: - family: << pipeline.parameters.ecs-service >> - cluster-name: << pipeline.parameters.ecs-cluster >> - container-image-name-updates: "container=transformers-mage,tag=${CIRCLE_SHA1}" - verify-revision-is-deployed: true - max-poll-attempts: 20 - poll-interval: 20 + # Apply all deployments + - run: + name: Deploy all indexers + command: | + # Replace IMAGE_TAG in all deployment files + for deployment in k8s/deployments/*.yaml; do + network=$(basename $deployment .yaml) + image_tag="${AWS_ECR_ACCOUNT_URL}/${pipeline.parameters.ecr-repository-prefix}:${CIRCLE_SHA1}-${network}" + + # Replace image tag in deployment file + sed -i "s|\${AWS_ACCOUNT_ID}.dkr.ecr.\${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-${network}|$image_tag|g" $deployment + + # Apply deployment + kubectl apply -f $deployment + + echo "Deployed $network" + done + + # Wait for deployments to be ready + for network in arbitrum-mainnet arbitrum-sepolia base-mainnet base-mainnet-lt base-sepolia eth-mainnet optimism-mainnet optimism-mainnet-tlx snax-mainnet snax-testnet; do + kubectl rollout status deployment/indexer-$network -n synthetix-data --timeout=300s + done + + build-transformers-mage: + docker: + - image: cimg/base:2023.03 + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - aws-ecr/build_and_push_image: + account_id: ${AWS_ACCOUNT_ID} + region: ${AWS_REGION} + repo: mage-synthetix-prod-image + tag: latest + build_path: ./transformers-mage + path: ./transformers-mage + dockerfile: Dockerfile + auth: + - aws-cli/setup: + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION workflows: version: 2 build-deploy: jobs: - - build-and-push: + - build-transformers-mage: context: aws-credentials filters: branches: - only: main - - deploy-to-ecs: + only: + - main + - feat/eks + - build-and-push-images: + context: aws-credentials + matrix: + parameters: + network: [ + "arbitrum-mainnet", + "arbitrum-sepolia", + "base-mainnet", + "base-mainnet-lt", + "base-sepolia", + "eth-mainnet", + "optimism-mainnet", + "optimism-mainnet-tlx", + "snax-mainnet", + "snax-testnet" + ] + filters: + branches: + only: + - main + - feat/eks + - deploy-to-eks: context: aws-credentials requires: - - build-and-push + - build-transformers-mage + - build-and-push-images filters: branches: - only: main \ No newline at end of file + only: main diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3c9b2f35..e5b03d60 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,20 +25,20 @@ jobs: - name: Install python dependencies run: | cd transformers && pip install -I --force-reinstall setuptools==71.0.0 && pip install --no-build-isolation --no-cache-dir -r requirements.txt - - name: Install dbt dependencies - run: | - cd transformers/synthetix && dbt deps - - name: Build docs - run: | - cd transformers/synthetix && dbt docs generate --profiles-dir profiles --profile synthetix-host - env: - PG_HOST: ${{ secrets.PG_HOST }} - PG_PASSWORD: ${{ secrets.PG_PASSWORD }} - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - with: - publish_branch: gh-pages - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: transformers/synthetix/target - force_orphan: true + # - name: Install dbt dependencies + # run: | + # cd transformers/synthetix && dbt deps + # - name: Build docs + # run: | + # cd transformers/synthetix && dbt docs generate --profiles-dir profiles --profile synthetix-host + # env: + # PG_HOST: ${{ secrets.PG_HOST }} + # PG_PASSWORD: ${{ secrets.PG_PASSWORD }} + # - name: Deploy to GitHub Pages + # uses: peaceiris/actions-gh-pages@v3 + # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + # with: + # publish_branch: gh-pages + # github_token: ${{ secrets.GITHUB_TOKEN }} + # publish_dir: transformers/synthetix/target + # force_orphan: true diff --git a/.gitignore b/.gitignore index 8d016ac0..c1723f87 100644 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,7 @@ pnpm-lock.yaml lib builds deployments +!k8s/deployments logs # local data @@ -184,4 +185,5 @@ secrets.toml transformers-mage/mage_data folder_contents.txt -.env.tmp \ No newline at end of file +.env.tmp +mage-ai.db diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 00000000..300766c5 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: db-config + namespace: synthetix-data +data: + # Aurora DB connection info - replace with actual values from your stack outputs + db-host: "synthetix-prod-postgres-cluster.cluster-xxxxxxxxxxxx.us-east-1.rds.amazonaws.com" + db-port: "5432" + + # Network RPC endpoints + arbitrum-mainnet-rpc: "https://arb1.arbitrum.io/rpc" + arbitrum-sepolia-rpc: "https://sepolia-rollup.arbitrum.io/rpc" + base-mainnet-rpc: "https://mainnet.base.org" + base-mainnet-lt-rpc: "https://mainnet.base.org" + base-sepolia-rpc: "https://sepolia.base.org" + eth-mainnet-rpc: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + optimism-mainnet-rpc: "https://mainnet.optimism.io" + optimism-mainnet-tlx-rpc: "https://mainnet.optimism.io" + snax-mainnet-rpc: "https://mainnet.snaxchain.io/" + snax-testnet-rpc: "https://testnet.snaxchain.io/" + \ No newline at end of file diff --git a/k8s/deployment-template.yaml b/k8s/deployment-template.yaml new file mode 100644 index 00000000..921e7f00 --- /dev/null +++ b/k8s/deployment-template.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-NETWORK_NAME + namespace: synthetix-data + labels: + app: synthetix-indexer + network: NETWORK_NAME +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: NETWORK_NAME + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: NETWORK_NAME + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-NETWORK_NAME + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "DB_NAME_VALUE" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: NETWORK_NAME-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/arbitrum-mainnet.yaml b/k8s/deployments/arbitrum-mainnet.yaml new file mode 100644 index 00000000..56acac04 --- /dev/null +++ b/k8s/deployments/arbitrum-mainnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-arbitrum-mainnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: arbitrum-mainnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: arbitrum-mainnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: arbitrum-mainnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-arbitrum-mainnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "arbitrum_mainnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: arbitrum-mainnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/arbitrum-sepolia.yaml b/k8s/deployments/arbitrum-sepolia.yaml new file mode 100644 index 00000000..ad1715df --- /dev/null +++ b/k8s/deployments/arbitrum-sepolia.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-arbitrum-sepolia + namespace: synthetix-data + labels: + app: synthetix-indexer + network: arbitrum-sepolia +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: arbitrum-sepolia + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: arbitrum-sepolia + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-arbitrum-sepolia + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "arbitrum_sepolia" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: arbitrum-sepolia-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/base-mainnet-lt.yaml b/k8s/deployments/base-mainnet-lt.yaml new file mode 100644 index 00000000..78e83579 --- /dev/null +++ b/k8s/deployments/base-mainnet-lt.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-base-mainnet-lt + namespace: synthetix-data + labels: + app: synthetix-indexer + network: base-mainnet-lt +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: base-mainnet-lt + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: base-mainnet-lt + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-base-mainnet-lt + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "base_mainnet_lt" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: base-mainnet-lt-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/base-mainnet.yaml b/k8s/deployments/base-mainnet.yaml new file mode 100644 index 00000000..928bdb52 --- /dev/null +++ b/k8s/deployments/base-mainnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-base-mainnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: base-mainnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: base-mainnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: base-mainnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-base-mainnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "base_mainnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: base-mainnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/base-sepolia.yaml b/k8s/deployments/base-sepolia.yaml new file mode 100644 index 00000000..d37c3f09 --- /dev/null +++ b/k8s/deployments/base-sepolia.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-base-sepolia + namespace: synthetix-data + labels: + app: synthetix-indexer + network: base-sepolia +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: base-sepolia + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: base-sepolia + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-base-sepolia + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "base_sepolia" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: base-sepolia-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/eth-mainnet.yaml b/k8s/deployments/eth-mainnet.yaml new file mode 100644 index 00000000..1b3c99f5 --- /dev/null +++ b/k8s/deployments/eth-mainnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-eth-mainnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: eth-mainnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: eth-mainnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: eth-mainnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-eth-mainnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "eth_mainnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: eth-mainnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/optimism-mainnet-tlx.yaml b/k8s/deployments/optimism-mainnet-tlx.yaml new file mode 100644 index 00000000..d8f6d1f0 --- /dev/null +++ b/k8s/deployments/optimism-mainnet-tlx.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-optimism-mainnet-tlx + namespace: synthetix-data + labels: + app: synthetix-indexer + network: optimism-mainnet-tlx +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: optimism-mainnet-tlx + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: optimism-mainnet-tlx + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-optimism-mainnet-tlx + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "optimism_mainnet_tlx" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: optimism-mainnet-tlx-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/optimism-mainnet.yaml b/k8s/deployments/optimism-mainnet.yaml new file mode 100644 index 00000000..fa781f12 --- /dev/null +++ b/k8s/deployments/optimism-mainnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-optimism-mainnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: optimism-mainnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: optimism-mainnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: optimism-mainnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-optimism-mainnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "optimism_mainnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: optimism-mainnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/snax-mainnet.yaml b/k8s/deployments/snax-mainnet.yaml new file mode 100644 index 00000000..f2e8c026 --- /dev/null +++ b/k8s/deployments/snax-mainnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-snax-mainnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: snax-mainnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: snax-mainnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: snax-mainnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-snax-mainnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "snax_mainnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: snax-mainnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/deployments/snax-testnet.yaml b/k8s/deployments/snax-testnet.yaml new file mode 100644 index 00000000..474dac87 --- /dev/null +++ b/k8s/deployments/snax-testnet.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-snax-testnet + namespace: synthetix-data + labels: + app: synthetix-indexer + network: snax-testnet +spec: + replicas: 1 + selector: + matchLabels: + app: synthetix-indexer + network: snax-testnet + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: synthetix-indexer + network: snax-testnet + spec: + containers: + - name: indexer + image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/synthetix-data/indexer:latest-snax-testnet + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + ports: + - containerPort: 4350 + name: gql + env: + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: db-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: db-config + key: db-port + - name: DB_NAME + value: "snax_testnet" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: password + - name: DB_USER + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: username + - name: GQL_PORT + value: "4350" + - name: RPC_ENDPOINT + valueFrom: + configMapKeyRef: + name: db-config + key: snax-testnet-rpc + - name: API_KEY + valueFrom: + secretKeyRef: + name: aurora-db-credentials + key: conduit-api-key + readinessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4350 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 60 diff --git a/k8s/external-secrets.yaml b/k8s/external-secrets.yaml new file mode 100644 index 00000000..6dc61cad --- /dev/null +++ b/k8s/external-secrets.yaml @@ -0,0 +1,30 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ClusterSecretStore +metadata: + name: aws-secretsmanager +spec: + provider: + aws: + service: SecretsManager + region: us-east-1 # Change to your AWS region + auth: + jwt: + serviceAccountRef: + name: external-secrets-sa + namespace: synthetix-data +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-secrets-sa + namespace: synthetix-data + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT_ID:role/EksExternalSecretsRole +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: indexer-service-account + namespace: synthetix-data + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT_ID:role/EksIndexerServiceRole \ No newline at end of file diff --git a/scripts/generate-deployments.sh b/scripts/generate-deployments.sh new file mode 100755 index 00000000..4702cf8f --- /dev/null +++ b/scripts/generate-deployments.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Create directories if they don't exist +mkdir -p k8s/deployments + +# First, create the template file +cat > k8s/deployment-template.yaml <