refactor(evaluators)!: reorganize into builtin + extra tiers

.githooks/pre-push

@@ -14,3 +14,12 @@ make lint
 
 echo "pre-push: running make typecheck"
 make typecheck
+
+# Check extras if they exist and have changes
+if [ -d "evaluators/extra/galileo" ]; then
+  echo "pre-push: checking evaluators/extra/galileo"
+  cd evaluators/extra/galileo
+  uv run --extra dev ruff check --config ../../../pyproject.toml src/
+  uv run --extra dev mypy --config-file ../../../pyproject.toml src/
+  cd "$REPO_ROOT"
+fi

.github/workflows/release.yaml

@@ -42,6 +42,7 @@ jobs:
           uv sync
           uv run python scripts/build.py all
 
+      # Publish in dependency order: models -> evaluators -> sdk -> evaluator-galileo
       - name: Publish agent-control-models to PyPI
         if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -50,6 +51,14 @@ jobs:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
+      - name: Publish agent-control-evaluators to PyPI
+        if: steps.release.outputs.released == 'true'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: evaluators/builtin/dist/
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
       - name: Publish agent-control-sdk to PyPI
         if: steps.release.outputs.released == 'true'
         uses: pypa/gh-action-pypi-publish@release/v1
@@ -58,6 +67,14 @@ jobs:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
+      - name: Publish agent-control-evaluator-galileo to PyPI
+        if: steps.release.outputs.released == 'true'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: evaluators/extra/galileo/dist/
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
       - name: Upload to GitHub Release
         if: steps.release.outputs.released == 'true'
         uses: python-semantic-release/upload-to-gh-release@main
@@ -67,5 +84,7 @@ jobs:
           root_options: "-vv"
           dist_glob: |
             models/dist/*
+            evaluators/builtin/dist/*
             sdks/python/dist/*
             server/dist/*
+            evaluators/extra/galileo/dist/*

AGENTS.md

@@ -26,7 +26,8 @@ Forwarded targets:
 - `engine/`: **control evaluation engine and evaluator system** — all evaluation logic, evaluator discovery, and evaluator orchestration lives here (`engine/src/agent_control_engine/`)
 - `server/`: FastAPI server (`server/src/agent_control_server/`)
 - `sdks/python/`: Python SDK — uses engine for evaluation (`sdks/python/src/agent_control/`)
-- `evaluators/`: evaluator implementations (`evaluators/src/agent_control_evaluators/`)
+- `evaluators/builtin/`: builtin evaluator implementations (`evaluators/builtin/src/agent_control_evaluators/`)
+- `evaluators/extra/`: optional evaluator packages (e.g., `evaluators/extra/galileo/`)
 - `ui/`: Nextjs based web app to manage agent controls 
 - `examples/`: runnable examples (ruff has relaxed import rules here)
 
@@ -66,13 +67,19 @@ All testing guidance (including “behavior changes require tests”) lives in `
   4) add SDK wrapper in `sdks/python/src/agent_control/`
   5) add tests (server + SDK) and update docs/examples if user-facing
 
-- Add a new evaluator:
-  1) implement evaluator class extending `Evaluator` in `evaluators/src/agent_control_evaluators/`
-  2) use `@register_evaluator` decorator (from `agent_control_models`)
-  3) add entry point in `evaluators/pyproject.toml` for auto-discovery
-  4) add tests in the evaluators package
+- Add a new builtin evaluator:
+  1) implement evaluator class extending `Evaluator` in `evaluators/builtin/src/agent_control_evaluators/`
+  2) use `@register_evaluator` decorator (from `agent_control_evaluators`)
+  3) add entry point in `evaluators/builtin/pyproject.toml` for auto-discovery
+  4) add tests in the evaluators/builtin package
   5) evaluator is automatically available to server and SDK via `discover_evaluators()`
 
+- Add an external evaluator package:
+  1) copy `evaluators/extra/template/` as a starting point
+  2) implement evaluator class extending `Evaluator` from `agent_control_evaluators`
+  3) add entry point using `org.name` format (e.g., `galileo.luna2`)
+  4) package is discovered automatically when installed alongside agent-control
+
 ## Git/PR workflow
 
 - Branch naming: `feature/...`, `fix/...`, `refactor/...`