feat/pdp shovel indexer

.devcontainer/Dockerfile

@@ -24,7 +24,9 @@ RUN apt-get update \
        mariadb-server mariadb-client \
     && rm -rf /var/lib/apt/lists/*
 
-# Prepare user-owned data dirs for rootless startup
+COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil
+
+# prepare user-owned data dirs for rootless startup
 RUN mkdir -p /home/vscode/.local/share/pg/pgdata \
     && mkdir -p /home/vscode/.local/share/mysql \
     && chown -R vscode:vscode /home/vscode/.local/share

.github/workflows/devcontainer-podman.yml

@@ -113,7 +113,7 @@ jobs:
           command: >
             cd /workspaces/singularity &&
             mkdir -p artifacts &&
-            SINGULARITY_TEST_INTEGRATION=true make test GOTESTSUM_ARGS="--junitfile artifacts/integration-tests.xml -- -timeout 20m -run Integration ./cmd/..."
+            SINGULARITY_TEST_INTEGRATION=true make test GOTESTSUM_ARGS="--junitfile artifacts/integration-tests.xml -- -timeout 20m -run Integration ./cmd/... ./service/pdptracker/..."
           container-runtime: podman
 
       - name: Report test results

cmd/run/pdptracker.go

@@ -2,55 +2,85 @@ package run
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/cockroachdb/errors"
+	"github.com/data-preservation-programs/go-synapse"
+	"github.com/data-preservation-programs/go-synapse/constants"
 	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service"
 	"github.com/data-preservation-programs/singularity/service/pdptracker"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/urfave/cli/v2"
 )
 
 var PDPTrackerCmd = &cli.Command{
 	Name:  "pdp-tracker",
-	Usage: "Start a PDP deal tracker that tracks f41 PDP deals for all relevant wallets",
-	Description: `The PDP tracker monitors Proof of Data Possession (PDP) deals on the Filecoin network.
-Unlike legacy f05 market deals, PDP deals use proof sets managed through the PDPVerifier contract
-where data is verified through cryptographic challenges.
-
-This tracker:
-- Monitors proof sets for tracked wallets
-- Updates deal status based on on-chain proof set state
-- Tracks challenge epochs and live status`,
+	Usage: "Track PDP deals via Shovel event indexing (requires PostgreSQL)",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
-			Name:     "eth-rpc",
-			Usage:    "Ethereum RPC endpoint for FEVM (e.g., https://api.node.glif.io)",
-			EnvVars:  []string{"ETH_RPC_URL"},
-			Required: true,
+			Name:    "eth-rpc",
+			Usage:   "Ethereum RPC endpoint for FEVM",
+			Value:   "https://api.node.glif.io/rpc/v1",
+			EnvVars: []string{"ETH_RPC_URL"},
 		},
 		&cli.DurationFlag{
 			Name:  "pdp-poll-interval",
-			Usage: "Polling interval for PDP transaction status",
+			Usage: "How often to check for new events in Shovel tables",
 			Value: 30 * time.Second,
 		},
+		&cli.BoolFlag{
+			Name:  "full-sync",
+			Usage: "Re-index all events from contract deployment (mainnet: block 5441432, calibnet: block 3140755). Requires an archival RPC node. Involves one RPC call per historical proof set.",
+		},
 	},
 	Action: func(c *cli.Context) error {
 		rpcURL := c.String("eth-rpc")
-		if rpcURL == "" {
-			return fmt.Errorf("eth-rpc is required")
+		connStr := c.String("database-connection-string")
+		if !strings.HasPrefix(connStr, "postgres:") && !strings.HasPrefix(connStr, "postgresql:") {
+			return errors.New("PDP tracking requires PostgreSQL (Shovel is Postgres-only)")
 		}
 
 		db, closer, err := database.OpenFromCLI(c)
 		if err != nil {
-			return err
+			return errors.WithStack(err)
 		}
 		defer closer.Close()
 
-		pdpClient, err := pdptracker.NewPDPClient(c.Context, rpcURL)
+		// detect network and contract address once, shared by indexer and rpc client
+		ethClient, err := ethclient.DialContext(c.Context, rpcURL)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "failed to connect to RPC")
+		}
+		network, chainID, err := synapse.DetectNetwork(c.Context, ethClient)
+		ethClient.Close()
+		if err != nil {
+			return errors.Wrap(err, "failed to detect network")
+		}
+
+		contractAddr := constants.GetPDPVerifierAddress(network)
+		if contractAddr == (common.Address{}) {
+			return fmt.Errorf("no PDPVerifier contract for network %s", network)
 		}
-		defer pdpClient.Close()
+
+		pdptracker.Logger.Infow("detected PDP network",
+			"network", network,
+			"chainId", chainID,
+			"contract", contractAddr.Hex(),
+		)
+
+		indexer, err := pdptracker.NewPDPIndexer(c.Context, connStr, rpcURL, uint64(chainID), contractAddr, c.Bool("full-sync"))
+		if err != nil {
+			return errors.Wrap(err, "failed to create PDP indexer")
+		}
+
+		rpcClient, err := pdptracker.NewPDPClient(c.Context, rpcURL, contractAddr)
+		if err != nil {
+			return errors.Wrap(err, "failed to create PDP RPC client")
+		}
+		defer rpcClient.Close()
 
 		cfg := pdptracker.PDPConfig{
 			PollingInterval: c.Duration("pdp-poll-interval"),
@@ -59,13 +89,8 @@ This tracker:
 			return err
 		}
 
-		tracker := pdptracker.NewPDPTracker(
-			db,
-			cfg,
-			pdpClient,
-			false,
-		)
+		tracker := pdptracker.NewPDPTracker(db, cfg, rpcClient, false)
 
-		return service.StartServers(c.Context, pdptracker.Logger, &tracker)
+		return service.StartServers(c.Context, pdptracker.Logger, indexer, &tracker)
 	},
 }

go.mod

@@ -8,8 +8,10 @@ require (
 	github.com/bcicen/jstream v1.0.1
 	github.com/brianvoe/gofakeit/v6 v6.23.2
 	github.com/cockroachdb/errors v1.11.3
+	github.com/data-preservation-programs/go-synapse v0.0.0-20260206105716-b6a5e7e6808e
 	github.com/data-preservation-programs/table v0.0.3
 	github.com/dustin/go-humanize v1.0.1
+	github.com/ethereum/go-ethereum v1.14.12
 	github.com/fatih/color v1.18.0
 	github.com/filecoin-project/go-address v1.2.0
 	github.com/filecoin-project/go-cbor-util v0.0.2
@@ -30,6 +32,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gotidy/ptr v1.4.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
+	github.com/indexsupply/shovel v0.1.9-0.20260111041930-aea8d42c335c
 	github.com/ipfs/boxo v0.35.0
 	github.com/ipfs/go-block-format v0.2.3
 	github.com/ipfs/go-cid v0.5.0
@@ -46,6 +49,7 @@ require (
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-trustless-utils v0.4.1
 	github.com/ipni/go-libipni v0.6.14
+	github.com/jackc/pgx/v5 v5.7.6
 	github.com/jellydator/ttlcache/v3 v3.0.1
 	github.com/joho/godotenv v1.5.1
 	github.com/jsign/go-filsigner v0.4.1
@@ -80,6 +84,7 @@ require (
 )
 
 require (
+	blake.io/pqx v0.2.1 // indirect
 	cloud.google.com/go/auth v0.17.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
@@ -145,7 +150,6 @@ require (
 	github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
 	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
-	github.com/data-preservation-programs/go-synapse v0.0.0-20260206105716-b6a5e7e6808e // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/dchest/blake2b v1.0.0 // indirect
@@ -162,7 +166,6 @@ require (
 	github.com/emersion/go-message v0.18.2 // indirect
 	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
 	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
-	github.com/ethereum/go-ethereum v1.14.12 // indirect
 	github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/filecoin-project/filecoin-ffi v1.34.0 // indirect
@@ -201,6 +204,7 @@ require (
 	github.com/go-openapi/spec v0.20.9 // indirect
 	github.com/go-resty/resty/v2 v2.16.5 // indirect
 	github.com/go-sql-driver/mysql v1.9.3 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/gofrs/flock v0.13.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
@@ -239,7 +243,6 @@ require (
 	github.com/ipni/index-provider v0.15.4 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.7.6 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
@@ -369,6 +372,7 @@ require (
 	github.com/wlynxg/anet v0.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -407,12 +411,14 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
+	kr.dev/errorfmt v0.1.1 // indirect
 	lukechampine.com/blake3 v1.4.1 // indirect
 	modernc.org/libc v1.22.3 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
 	modernc.org/memory v1.5.0 // indirect
 	modernc.org/sqlite v1.21.1 // indirect
 	moul.io/http2curl v1.0.0 // indirect
+	nhooyr.io/websocket v1.8.10 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 	storj.io/common v0.0.0-20251022143549-19bf6a9f274a // indirect
 	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect