Leo/v0.46 ledger service

.github/workflows/ci.yml

@@ -328,6 +328,7 @@ jobs:
           gcr.io/flow-container-registry/collection:latest \
           gcr.io/flow-container-registry/consensus:latest \
           gcr.io/flow-container-registry/execution:latest \
+          gcr.io/flow-container-registry/execution-ledger:latest \
           gcr.io/flow-container-registry/ghost:latest \
           gcr.io/flow-container-registry/observer:latest \
           gcr.io/flow-container-registry/verification:latest \

.github/workflows/image_builds.yml

@@ -1,4 +1,4 @@
-name: Build & Promote Docker Images to Public Registry 
+name: Build & Promote Docker Images to Public Registry
 on:
   workflow_dispatch:
     inputs:
@@ -25,53 +25,59 @@ jobs:
   # The environment is set to 'container builds' that provides the necessary secrets for pushing to the pirvate registry.
   public-build:
     if: ${{ github.event.inputs.secure-build == 'false' }}
-    name: Execute public repo build & push to private artifact registry 
+    name: Execute public repo build & push to private artifact registry
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        # We specify all of the potential build commands for each role. 
+        # We specify all of the potential build commands for each role.
         # This allows us to build and push all images in parallel, reducing the overall build time.
         # The matrix is defined to include all roles & image types that we want to build and push.
         # These commands are targets defined in the Makefile of the repository.
         build_command:
           # access Build Commands
-          - docker-build-access-with-adx docker-push-access-with-adx 
-          - docker-build-access-without-adx docker-push-access-without-adx 
-          - docker-build-access-without-netgo-without-adx docker-push-access-without-netgo-without-adx 
-          - docker-cross-build-access-arm docker-push-access-arm 
+          - docker-build-access-with-adx docker-push-access-with-adx
+          - docker-build-access-without-adx docker-push-access-without-adx
+          - docker-build-access-without-netgo-without-adx docker-push-access-without-netgo-without-adx
+          - docker-cross-build-access-arm docker-push-access-arm
 
           # collection Build Commands
-          - docker-build-collection-with-adx docker-push-collection-with-adx 
-          - docker-build-collection-without-adx docker-push-collection-without-adx 
-          - docker-build-collection-without-netgo-without-adx docker-push-collection-without-netgo-without-adx 
-          - docker-cross-build-collection-arm docker-push-collection-arm 
+          - docker-build-collection-with-adx docker-push-collection-with-adx
+          - docker-build-collection-without-adx docker-push-collection-without-adx
+          - docker-build-collection-without-netgo-without-adx docker-push-collection-without-netgo-without-adx
+          - docker-cross-build-collection-arm docker-push-collection-arm
 
           # consensus Build Commands
-          - docker-build-consensus-with-adx docker-push-consensus-with-adx 
-          - docker-build-consensus-without-adx docker-push-consensus-without-adx 
-          - docker-build-consensus-without-netgo-without-adx docker-push-consensus-without-netgo-without-adx 
-          - docker-cross-build-consensus-arm docker-push-consensus-arm 
+          - docker-build-consensus-with-adx docker-push-consensus-with-adx
+          - docker-build-consensus-without-adx docker-push-consensus-without-adx
+          - docker-build-consensus-without-netgo-without-adx docker-push-consensus-without-netgo-without-adx
+          - docker-cross-build-consensus-arm docker-push-consensus-arm
 
           # execution Build Commands
-          - docker-build-execution-with-adx docker-push-execution-with-adx 
-          - docker-build-execution-without-adx docker-push-execution-without-adx 
-          - docker-build-execution-without-netgo-without-adx docker-push-execution-without-netgo-without-adx 
-          - docker-cross-build-execution-arm docker-push-execution-arm 
+          - docker-build-execution-with-adx docker-push-execution-with-adx
+          - docker-build-execution-without-adx docker-push-execution-without-adx
+          - docker-build-execution-without-netgo-without-adx docker-push-execution-without-netgo-without-adx
+          - docker-cross-build-execution-arm docker-push-execution-arm
+
+          # execution Ledger Service Build Commands
+          - docker-build-execution-ledger-with-adx docker-push-execution-ledger-with-adx
+          - docker-build-execution-ledger-without-adx docker-push-execution-ledger-without-adx
+          - docker-build-execution-ledger-without-netgo-without-adx docker-push-execution-ledger-without-netgo-without-adx
+          - docker-cross-build-execution-ledger-arm docker-push-execution-ledger-arm
 
           # observer Build Commands
-          - docker-build-observer-with-adx docker-push-observer-with-adx 
-          - docker-build-observer-without-adx docker-push-observer-without-adx 
-          - docker-build-observer-without-netgo-without-adx docker-push-observer-without-netgo-without-adx 
-          - docker-cross-build-observer-arm docker-push-observer-arm 
+          - docker-build-observer-with-adx docker-push-observer-with-adx
+          - docker-build-observer-without-adx docker-push-observer-without-adx
+          - docker-build-observer-without-netgo-without-adx docker-push-observer-without-netgo-without-adx
+          - docker-cross-build-observer-arm docker-push-observer-arm
 
           # verification Build Commands
-          - docker-build-verification-with-adx docker-push-verification-with-adx 
-          - docker-build-verification-without-adx docker-push-verification-without-adx 
-          - docker-build-verification-without-netgo-without-adx docker-push-verification-without-netgo-without-adx 
-          - docker-cross-build-verification-arm docker-push-verification-arm 
+          - docker-build-verification-with-adx docker-push-verification-with-adx
+          - docker-build-verification-without-adx docker-push-verification-without-adx
+          - docker-build-verification-without-netgo-without-adx docker-push-verification-without-netgo-without-adx
+          - docker-cross-build-verification-arm docker-push-verification-arm
 
-    environment: container builds 
+    environment: container builds
     steps:
       - name: Setup Go
         uses: actions/setup-go@v4
@@ -119,7 +125,7 @@ jobs:
       fail-fast: false
       matrix:
         role: [access, collection, consensus, execution, observer, verification]
-    environment: secure builds 
+    environment: secure builds
     steps:
       - uses: actions/create-github-app-token@v2
         id: app-token
@@ -133,8 +139,8 @@ jobs:
           client_payload: '{"role": "${{ matrix.role }}", "tag": "${{ inputs.tag }}"}'
           github_token: ${{ steps.app-token.outputs.token }}
           owner: 'onflow'
-          repo: ${{ secrets.SECURE_BUILDS_REPO }}  
-          ref: master-private 
+          repo: ${{ secrets.SECURE_BUILDS_REPO }}
+          ref: master-private
           workflow_file_name: 'secure_build.yml'
 
   promote-to-partner-registry:
@@ -155,19 +161,19 @@ jobs:
       fail-fast: false
       matrix:
         role: [access]
-    environment: ${{ matrix.role }} image promotion to partner registry 
+    environment: ${{ matrix.role }} image promotion to partner registry
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
 
-      - name: Promote ${{ matrix.role }} 
+      - name: Promote ${{ matrix.role }}
         uses: ./actions/promote-images
         with:
           gcp_credentials: ${{ secrets.PARTNER_REGISTRY_PROMOTION_SECRET }}
           private_registry: ${{ vars.PRIVATE_REGISTRY }}
           private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }}
           promotion_registry: ${{ vars.PARTNER_REGISTRY }}
-          role: ${{ matrix.role }} 
+          role: ${{ matrix.role }}
           tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm"
 
   promote-to-public-registry:
@@ -187,18 +193,18 @@ jobs:
       fail-fast: false
       matrix:
         role: [access, collection, consensus, execution, observer, verification]
-    environment: ${{ matrix.role }} image promotion to public registry 
+    environment: ${{ matrix.role }} image promotion to public registry
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
 
-      - name: Promote ${{ matrix.role }} 
+      - name: Promote ${{ matrix.role }}
         uses: ./actions/promote-images
         with:
           gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }}
           private_registry: ${{ vars.PRIVATE_REGISTRY }}
           private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }}
           promotion_registry: ${{ vars.PUBLIC_REGISTRY }}
-          role: ${{ matrix.role }} 
+          role: ${{ matrix.role }}
           tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm"
 

CONTRIBUTING.md

@@ -109,17 +109,11 @@ following when creating your pull request:
 
 A reviewer will be assigned automatically when your PR is created.
 
-We use [bors](https://github.com/bors-ng/bors-ng) merge bot to ensure that the `master` branch never breaks.
-Once a PR is approved, you can comment on it with the following to add your PR to the merge queue:
+We use GitHub Actions to ensure that the `master` branch never breaks.
+Once a PR is approved and CI passes, you can add it to the merge queue.
+If the PR fails in the merge queue, you will need to fix it and try again.
 
-```
-bors merge
-```
-
-If the PR passes CI, it will automatically be pushed to the `master` branch. If it fails, bors will comment
-on the PR so you can fix it.
-
-See the [documentation](https://bors.tech/documentation/) for a more comprehensive list of bors commands.
+See GitHub's [merge queue documentation](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue) for more details.
 
 ## Style Guide
 

Makefile

@@ -145,7 +145,7 @@ generate: generate-proto generate-mocks generate-fvm-env-wrappers
 
 .PHONY: generate-proto
 generate-proto:
-	prototool generate protobuf
+	cd ledger/protobuf && buf generate
 
 .PHONY: generate-fvm-env-wrappers
 generate-fvm-env-wrappers:
@@ -371,6 +371,42 @@ docker-cross-build-execution-arm:
 		--label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \
 		-t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_ARM)" .
 
+.PHONY: docker-build-execution-ledger-with-adx
+docker-build-execution-ledger-with-adx:
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ledger --build-arg COMMIT=$(COMMIT)  --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
+		--label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG)" .
+
+.PHONY: docker-build-execution-ledger-without-adx
+docker-build-execution-ledger-without-adx:
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ledger --build-arg COMMIT=$(COMMIT)  --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
+		--label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_NO_ADX)" .
+
+.PHONY: docker-build-execution-ledger-without-netgo-without-adx
+docker-build-execution-ledger-without-netgo-without-adx:
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ledger --build-arg COMMIT=$(COMMIT)  --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
+		--label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_NO_NETGO_NO_ADX)" .
+
+.PHONY: docker-cross-build-execution-ledger-arm
+docker-cross-build-execution-ledger-arm:
+	docker build -f cmd/Dockerfile  --build-arg TARGET=./cmd/ledger --build-arg COMMIT=$(COMMIT)  --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
+		--label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_ARM)" .
+
+.PHONY: docker-native-build-execution-ledger
+docker-native-build-execution-ledger:
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ledger --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
+		--label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:latest" \
+		-t "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG)" .
+
 .PHONY: docker-native-build-execution-debug
 docker-native-build-execution-debug:
 	docker build -f cmd/Dockerfile  --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT)  --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \
@@ -558,7 +594,7 @@ docker-native-build-ghost-debug:
 		-t "$(CONTAINER_REGISTRY)/ghost-debug:latest" \
 		-t "$(CONTAINER_REGISTRY)/ghost-debug:$(IMAGE_TAG)" .
 
-PHONY: docker-build-bootstrap
+.PHONY: docker-build-bootstrap
 docker-build-bootstrap:
 	docker build -f cmd/Dockerfile  --build-arg TARGET=./cmd/bootstrap --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \
 		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY \
@@ -590,7 +626,7 @@ docker-native-build-loader:
 		-t "$(CONTAINER_REGISTRY)/loader:$(IMAGE_TAG)" .
 
 .PHONY: docker-native-build-flow
-docker-native-build-flow: docker-native-build-collection docker-native-build-consensus docker-native-build-execution docker-native-build-verification docker-native-build-access docker-native-build-observer docker-native-build-ghost
+docker-native-build-flow: docker-native-build-collection docker-native-build-consensus docker-native-build-execution docker-native-build-execution-ledger docker-native-build-verification docker-native-build-access docker-native-build-observer docker-native-build-ghost
 
 .PHONY: docker-build-flow-with-adx
 docker-build-flow-with-adx: docker-build-collection-with-adx docker-build-consensus-with-adx docker-build-execution-with-adx docker-build-verification-with-adx docker-build-access-with-adx docker-build-observer-with-adx
@@ -676,6 +712,22 @@ docker-push-execution-arm:
 docker-push-execution-latest: docker-push-execution
 	docker push "$(CONTAINER_REGISTRY)/execution:latest"
 
+.PHONY: docker-push-execution-ledger-with-adx
+docker-push-execution-ledger-with-adx:
+	docker push "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG)"
+
+.PHONY: docker-push-execution-ledger-without-adx
+docker-push-execution-ledger-without-adx:
+	docker push "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_NO_ADX)"
+
+.PHONY: docker-push-execution-ledger-without-netgo-without-adx
+docker-push-execution-ledger-without-netgo-without-adx:
+	docker push "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_NO_NETGO_NO_ADX)"
+
+.PHONY: docker-push-execution-ledger-arm
+docker-push-execution-ledger-arm:
+	docker push "$(CONTAINER_REGISTRY)/execution-ledger:$(IMAGE_TAG_ARM)"
+
 .PHONY: docker-push-verification-with-adx
 docker-push-verification-with-adx:
 	docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)"

admin/buf.lock

@@ -1,9 +1,8 @@
 # Generated by buf. DO NOT EDIT.
+version: v1beta1
 deps:
   - remote: buf.build
     owner: googleapis
     repository: googleapis
-    branch: main
-    commit: 04ad98c82478417784639b43e71c6b4c
-    digest: b1-8nhYmpcJRqI1lyfXpbPH_nQjQfzgGoVHXq_gA7E4mjg=
-    create_time: 2021-09-07T16:08:38.569839Z
+    commit: 004180b77378443887d3b55cabc00384
+    digest: shake256:d26c7c2fd95f0873761af33ca4a0c0d92c8577122b6feb74eb3b0a57ebe47a98ab24a209a0e91945ac4c77204e9da0c2de0020b2cedc27bdbcdea6c431eec69b

admin/command_runner.go

@@ -195,6 +195,13 @@ func (r *CommandRunner) runAdminServer(ctx irrecoverable.SignalerContext) error
 
 	r.logger.Info().Msg("admin server starting up")
 
+	// Remove stale socket file from previous run (e.g. after container/process restart)
+	if _, err := os.Stat(r.grpcAddress); err == nil {
+		if removeErr := os.Remove(r.grpcAddress); removeErr != nil {
+			r.logger.Warn().Err(removeErr).Str("socket", r.grpcAddress).Msg("failed to remove stale admin socket")
+		}
+	}
+
 	listener, err := net.Listen("unix", r.grpcAddress)
 	if err != nil {
 		return fmt.Errorf("failed to listen on admin server address: %w", err)

admin/commands/execution/checkpoint_trigger.go

@@ -13,14 +13,25 @@ import (
 var _ commands.AdminCommand = (*TriggerCheckpointCommand)(nil)
 
 // TriggerCheckpointCommand will send a signal to compactor to trigger checkpoint
-// once finishing writing the current WAL segment file
+// once finishing writing the current WAL segment file.
+// When running in remote ledger mode (ledgerServiceAddr is non-empty), this command
+// returns an error directing users to the ledger service's admin endpoint.
 type TriggerCheckpointCommand struct {
-	trigger *atomic.Bool
+	trigger                *atomic.Bool
+	ledgerServiceAddr      string // non-empty when using remote ledger service
+	ledgerServiceAdminAddr string // admin HTTP address for remote ledger service
 }
 
-func NewTriggerCheckpointCommand(trigger *atomic.Bool) *TriggerCheckpointCommand {
+// NewTriggerCheckpointCommand creates a new TriggerCheckpointCommand.
+// Parameters:
+//   - trigger: atomic bool to signal the compactor (used only in local ledger mode)
+//   - ledgerServiceAddr: gRPC address of the remote ledger service (empty string for local mode)
+//   - ledgerServiceAdminAddr: admin HTTP address of the remote ledger service (for error messages)
+func NewTriggerCheckpointCommand(trigger *atomic.Bool, ledgerServiceAddr, ledgerServiceAdminAddr string) *TriggerCheckpointCommand {
 	return &TriggerCheckpointCommand{
-		trigger: trigger,
+		trigger:                trigger,
+		ledgerServiceAddr:      ledgerServiceAddr,
+		ledgerServiceAdminAddr: ledgerServiceAdminAddr,
 	}
 }
 
@@ -35,5 +46,23 @@ func (s *TriggerCheckpointCommand) Handler(_ context.Context, _ *admin.CommandRe
 }
 
 func (s *TriggerCheckpointCommand) Validator(_ *admin.CommandRequest) error {
+	// When using remote ledger service, checkpointing is handled by the ledger service
+	if s.ledgerServiceAddr != "" {
+		if s.ledgerServiceAdminAddr == "" {
+			return admin.NewInvalidAdminReqErrorf(
+				"trigger-checkpoint is not available when using remote ledger service (connected to %s). "+
+					"Please use the ledger service's admin endpoint instead. "+
+					"The admin address was not configured - check if the ledger service was started with --admin-addr",
+				s.ledgerServiceAddr,
+			)
+		}
+		return admin.NewInvalidAdminReqErrorf(
+			"trigger-checkpoint is not available when using remote ledger service (connected to %s). "+
+				"Please use the ledger service's admin endpoint instead: "+
+				"curl -X POST http://%s/admin/run_command -H 'Content-Type: application/json' -d '{\"commandName\": \"trigger-checkpoint\", \"data\": {}}'",
+			s.ledgerServiceAddr,
+			s.ledgerServiceAdminAddr,
+		)
+	}
 	return nil
 }