feat: snapshot CLI, Airbyte pull, fixture seeding, Jira + Salesforce services

.github/workflows/ci.yml

@@ -23,6 +23,17 @@ jobs:
       - name: Check formatting
         run: cargo fmt --all -- --check
 
+  python-unit-tests:
+    name: Python Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run snapshot_pull tests
+        run: python -m unittest discover -s services/_lib/tests -v
+
   clippy:
     name: Clippy
     runs-on: ubuntu-latest
@@ -68,10 +79,10 @@ jobs:
       services: ${{ steps.discover.outputs.services }}
     steps:
       - uses: actions/checkout@v4
-      - name: Find services with service.yaml
+      - name: Find services with contract tests
         id: discover
         run: |
-          services=$(ls -d services/*/service.yaml 2>/dev/null | xargs -I {} dirname {} | xargs -I {} basename {} | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          services=$(for d in services/*/; do [ -f "$d/service.yaml" ] && [ -d "$d/contracts" ] && basename "$d"; done | jq -R -s -c 'split("\n") | map(select(length > 0))')
           echo "services=$services" >> $GITHUB_OUTPUT
 
   contracts:

.gitignore

@@ -1 +1,21 @@
-/target
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.pytest_cache/
+
+# Rust
+/target/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# DoubleAgent
+.doubleagent.env

README.md

@@ -46,7 +46,14 @@ doubleagent start github slack        # Multiple services
 doubleagent status                    # Show running services
 doubleagent stop                      # Stop all
 doubleagent reset github              # Clear state
-doubleagent seed github ./data.yaml   # Load fixtures
+doubleagent seed github ./data.yaml         # Load from file
+doubleagent seed github --fixture startup   # Load from service fixtures/ dir
+doubleagent seed github --snapshot default  # Seed from pulled snapshot
+
+doubleagent snapshot pull github      # Pull snapshot from configured connector
+doubleagent snapshot list             # List local snapshots
+doubleagent snapshot inspect github -p default
+doubleagent snapshot delete github -p default
 ```
 
 When a service starts, the CLI prints the environment variable to use:
@@ -75,6 +82,24 @@ repo = client.get_user().create_repo("test-repo")
 issue = repo.create_issue(title="Test issue")
 ```
 
+### Snapshot Pulls (Airbyte)
+
+`doubleagent snapshot pull <service>` reads connector config from `service.yaml`, pulls
+records from Airbyte, writes a local snapshot profile, and produces a seed payload at:
+
+`~/.doubleagent/snapshots/<service>/<profile>/seed.json`
+
+Typical workflow:
+
+```bash
+doubleagent start github
+doubleagent snapshot pull github --profile default
+doubleagent seed github --snapshot default
+```
+
+Snapshot pulls are gated by `connector.required_env` in the service definition and run with
+redaction by default.
+
 ## Project Configuration
 
 Define which services your project needs in a `doubleagent.yaml` file at the root of your repository:
@@ -135,10 +160,11 @@ doubleagent start github slack
 | GitHub | ✅ Available | PyGithub, octokit |
 | Slack | ✅ Available | slack_sdk |
 | Descope | ✅ Available | descope |
-| Jira | 🚧 Coming soon | atlassian-python-api |
-| Okta | 🚧 Coming soon | okta |
 | Auth0 | ✅ Available | auth0-python |
 | Stripe | ✅ Available | stripe |
+| Jira | ✅ Available (snapshot) | atlassian-python-api |
+| Salesforce | ✅ Available (snapshot) | simple-salesforce |
+| Okta | 🚧 Coming soon | okta |
 
 ## Contributing
 

crates/cli/src/commands/mod.rs

@@ -4,6 +4,7 @@ pub mod list;
 pub mod reset;
 pub mod run;
 pub mod seed;
+pub mod snapshot;
 pub mod start;
 pub mod status;
 pub mod stop;
@@ -50,6 +51,9 @@ pub enum Commands {
 
     /// Run a command with services started and env vars set
     Run(RunArgs),
+
+    /// Pull and manage snapshot profiles
+    Snapshot(SnapshotArgs),
 }
 
 #[derive(Parser)]
@@ -107,7 +111,15 @@ pub struct SeedArgs {
     pub service: String,
 
     /// Path to seed data file (YAML or JSON)
-    pub file: String,
+    pub file: Option<String>,
+
+    /// Seed from snapshot profile (uses ~/.doubleagent/snapshots/<service>/<profile>/seed.json)
+    #[arg(long)]
+    pub snapshot: Option<String>,
+
+    /// Name of a fixture in the service's fixtures/ directory (e.g. "startup")
+    #[arg(long)]
+    pub fixture: Option<String>,
 }
 
 #[derive(Parser)]
@@ -134,3 +146,73 @@ pub struct RunArgs {
     #[arg(last = true, required = true)]
     pub command: Vec<String>,
 }
+
+#[derive(Parser)]
+pub struct SnapshotArgs {
+    #[command(subcommand)]
+    pub command: SnapshotCommands,
+}
+
+#[derive(Subcommand)]
+pub enum SnapshotCommands {
+    /// Pull a snapshot profile from a configured connector
+    Pull(SnapshotPullArgs),
+    /// List available snapshot profiles
+    List(SnapshotListArgs),
+    /// Print a snapshot manifest as JSON
+    Inspect(SnapshotInspectArgs),
+    /// Delete a snapshot profile
+    Delete(SnapshotDeleteArgs),
+}
+
+#[derive(Parser)]
+pub struct SnapshotPullArgs {
+    /// Service to pull a snapshot for
+    pub service: String,
+
+    /// Snapshot profile name
+    #[arg(long, short)]
+    pub profile: Option<String>,
+
+    /// Max records per stream
+    #[arg(long, short)]
+    pub limit: Option<u32>,
+
+    /// Disable redaction
+    #[arg(long)]
+    pub no_redact: bool,
+
+    /// Merge into existing snapshot profile by id
+    #[arg(long)]
+    pub incremental: bool,
+
+    /// Force backend ("pyairbyte")
+    #[arg(long)]
+    pub backend: Option<String>,
+}
+
+#[derive(Parser)]
+pub struct SnapshotListArgs {
+    /// Filter by service
+    pub service: Option<String>,
+}
+
+#[derive(Parser)]
+pub struct SnapshotInspectArgs {
+    /// Service name
+    pub service: String,
+
+    /// Snapshot profile name
+    #[arg(long, short)]
+    pub profile: String,
+}
+
+#[derive(Parser)]
+pub struct SnapshotDeleteArgs {
+    /// Service name
+    pub service: String,
+
+    /// Snapshot profile name
+    #[arg(long, short)]
+    pub profile: String,
+}