diff --git a/.dockerignore b/.dockerignore index 6b0abd2..776cc30 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,7 +17,7 @@ testdata/ # Build artifacts build/ -dhamps-vdb +embapi # Go workspace go.work diff --git a/.env.docker.template b/.env.docker.template index e0b91ee..c9fdb85 100644 --- a/.env.docker.template +++ b/.env.docker.template @@ -1,8 +1,8 @@ # ========================================== -# dhamps-vdb Docker Environment Configuration +# embapi Docker Environment Configuration # ========================================== # -# This file contains environment variables for running dhamps-vdb with Docker. +# This file contains environment variables for running embapi with Docker. # Copy this file to .env and update the values as needed. # # IMPORTANT: Never commit the actual .env file to version control! @@ -21,7 +21,7 @@ SERVICE_DBHOST=postgres SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=your_secure_password_here -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi # Port Mapping # ------------ diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2bab141..1376059 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -41,7 +41,7 @@ jobs: run: go mod download - name: Verify build - run: go build -o build/dhamps-vdb main.go + run: go build -o build/embapi main.go - name: Run tests run: go test -v ./... diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 818b739..51af270 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: run: go mod download - name: Verify build - run: go build -o build/dhamps-vdb main.go + run: go build -o build/embapi main.go - name: Run tests run: go test -v ./... diff --git a/.gitignore b/.gitignore index f86ba5b..d4c32c0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *.so *.dylib build/* +embapi # Test binary, built with `go test -c` *.test diff --git a/DOCKER.md b/DOCKER.md index 643ebed..f531f30 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -1,6 +1,6 @@ # Docker Deployment Guide -This guide explains how to deploy dhamps-vdb using Docker containers. +This guide explains how to deploy embapi using Docker containers. ## Table of Contents @@ -17,7 +17,7 @@ This guide explains how to deploy dhamps-vdb using Docker containers. ## Quick Start -The fastest way to get started with dhamps-vdb using Docker: +The fastest way to get started with embapi using Docker: ### Automated Setup (Recommended) @@ -29,7 +29,7 @@ The fastest way to get started with dhamps-vdb using Docker: docker-compose up -d # Check logs -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi # Access the API curl http://localhost:8880/docs @@ -50,7 +50,7 @@ cp .env.docker.template .env docker-compose up -d # 4. Check logs -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi # 5. Access the API curl http://localhost:8880/docs @@ -82,7 +82,7 @@ All configuration is done via environment variables. You can set them: | `SERVICE_DBPORT` | Database port | `5432` | | `SERVICE_DBUSER` | Database username | `postgres` | | `SERVICE_DBPASSWORD` | Database password | `postgres` | -| `SERVICE_DBNAME` | Database name | `dhamps_vdb` | +| `SERVICE_DBNAME` | Database name | `embapi` | | `API_PORT` | External port to expose API | `8880` | | `POSTGRES_PORT` | External port to expose PostgreSQL | `5432` | @@ -90,7 +90,7 @@ All configuration is done via environment variables. You can set them: ### Option 1: Docker Compose with Included PostgreSQL (Recommended) -This is the easiest option for getting started. It includes both dhamps-vdb and a PostgreSQL database with pgvector support. +This is the easiest option for getting started. It includes both embapi and a PostgreSQL database with pgvector support. ```bash docker-compose up -d @@ -98,21 +98,21 @@ docker-compose up -d This will: - Start a PostgreSQL 16 container with pgvector extension -- Build and start the dhamps-vdb container +- Build and start the embapi container - Set up networking between the containers - Persist database data in a Docker volume ### Option 2: Standalone Container with External Database -If you have an existing PostgreSQL database with pgvector support, you can run just the dhamps-vdb container: +If you have an existing PostgreSQL database with pgvector support, you can run just the embapi container: ```bash # Build the image -docker build -t dhamps-vdb:latest . +docker build -t embapi:latest . # Run the container docker run -d \ - --name dhamps-vdb \ + --name embapi \ -p 8880:8880 \ -e SERVICE_DBHOST=your-db-host \ -e SERVICE_DBPORT=5432 \ @@ -121,7 +121,7 @@ docker run -d \ -e SERVICE_DBNAME=your-db-name \ -e SERVICE_ADMINKEY=your-admin-key \ -e ENCRYPTION_KEY=your-encryption-key \ - dhamps-vdb:latest + embapi:latest ``` ### Option 3: Docker Compose with External Database @@ -135,7 +135,7 @@ Modify `docker-compose.yml` to comment out or remove the `postgres` service, the # ... services: - dhamps-vdb: + embapi: # ... rest of the configuration environment: SERVICE_DBHOST: external-db.example.com # Your external database @@ -148,13 +148,13 @@ To build the Docker image manually: ```bash # Build with default tag -docker build -t dhamps-vdb:latest . +docker build -t embapi:latest . # Build with specific tag -docker build -t dhamps-vdb:v0.1.0 . +docker build -t embapi:v0.1.0 . # Build with no cache (clean build) -docker build --no-cache -t dhamps-vdb:latest . +docker build --no-cache -t embapi:latest . ``` The Dockerfile uses a multi-stage build: @@ -186,7 +186,7 @@ docker-compose logs docker-compose logs -f # View logs for specific service -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi docker-compose logs -f postgres ``` @@ -210,7 +210,7 @@ docker-compose down -v docker-compose restart # Restart specific service -docker-compose restart dhamps-vdb +docker-compose restart embapi ``` ## Running Standalone Container @@ -219,17 +219,17 @@ docker-compose restart dhamps-vdb ```bash docker run -d \ - --name dhamps-vdb \ + --name embapi \ -p 8880:8880 \ --env-file .env \ - dhamps-vdb:latest + embapi:latest ``` ### Run with Individual Environment Variables ```bash docker run -d \ - --name dhamps-vdb \ + --name embapi \ -p 8880:8880 \ -e SERVICE_DEBUG=false \ -e SERVICE_HOST=0.0.0.0 \ @@ -238,20 +238,20 @@ docker run -d \ -e SERVICE_DBPORT=5432 \ -e SERVICE_DBUSER=dbuser \ -e SERVICE_DBPASSWORD=dbpass \ - -e SERVICE_DBNAME=dhamps_vdb \ + -e SERVICE_DBNAME=embapi \ -e SERVICE_ADMINKEY=admin-key \ -e ENCRYPTION_KEY=encryption-key-32-chars-min \ - dhamps-vdb:latest + embapi:latest ``` ### Interactive Run (for debugging) ```bash docker run -it --rm \ - --name dhamps-vdb \ + --name embapi \ -p 8880:8880 \ --env-file .env \ - dhamps-vdb:latest + embapi:latest ``` ## Connecting to External Database @@ -270,34 +270,34 @@ Connect to your PostgreSQL server and run: ```sql -- Create database -CREATE DATABASE dhamps_vdb; +CREATE DATABASE embapi; -- Create user -CREATE USER dhamps_user WITH PASSWORD 'secure_password'; +CREATE USER embapi_user WITH PASSWORD 'secure_password'; -- Grant privileges -GRANT ALL PRIVILEGES ON DATABASE dhamps_vdb TO dhamps_user; +GRANT ALL PRIVILEGES ON DATABASE embapi TO embapi_user; -- Connect to the database -\c dhamps_vdb +\c embapi -- Grant schema permissions -GRANT ALL ON SCHEMA public TO dhamps_user; +GRANT ALL ON SCHEMA public TO embapi_user; -- Enable pgvector extension CREATE EXTENSION IF NOT EXISTS vector; ``` -### Configure dhamps-vdb +### Configure embapi In your `.env` file: ```bash SERVICE_DBHOST=your-database-host.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi ``` For docker-compose with external database, you can either: @@ -309,7 +309,7 @@ For docker-compose with external database, you can either: # ... ``` -2. **Or skip docker-compose** and run just the dhamps-vdb container with `docker run`. +2. **Or skip docker-compose** and run just the embapi container with `docker run`. ## Data Persistence @@ -329,17 +329,17 @@ docker volume ls Inspect volume: ```bash -docker volume inspect dhamps-vdb_postgres_data +docker volume inspect embapi_postgres_data ``` ### Backup Database ```bash # Backup -docker-compose exec postgres pg_dump -U postgres dhamps_vdb > backup.sql +docker-compose exec postgres pg_dump -U postgres embapi > backup.sql # Restore -docker-compose exec -T postgres psql -U postgres dhamps_vdb < backup.sql +docker-compose exec -T postgres psql -U postgres embapi < backup.sql ``` ## Security Considerations @@ -373,7 +373,7 @@ For Docker Swarm: ```yaml services: - dhamps-vdb: + embapi: secrets: - admin_key - encryption_key @@ -394,8 +394,8 @@ secrets: Check logs: ```bash -docker-compose logs dhamps-vdb -docker logs dhamps-vdb +docker-compose logs embapi +docker logs embapi ``` Common issues: @@ -413,7 +413,7 @@ docker-compose ps docker-compose logs postgres # Test database connection -docker-compose exec postgres psql -U postgres -d dhamps_vdb -c "SELECT 1;" +docker-compose exec postgres psql -U postgres -d embapi -c "SELECT 1;" ``` ### Can't connect to API @@ -423,10 +423,10 @@ docker-compose exec postgres psql -U postgres -d dhamps_vdb -c "SELECT 1;" docker ps # Check port mapping -docker port dhamps-vdb +docker port embapi # Test from inside container -docker-compose exec dhamps-vdb wget -O- http://localhost:8880/docs +docker-compose exec embapi wget -O- http://localhost:8880/docs # Test from host curl http://localhost:8880/docs @@ -445,7 +445,7 @@ If you see permission errors, check: docker-compose down -v # Remove images -docker rmi dhamps-vdb:latest +docker rmi embapi:latest docker rmi pgvector/pgvector:0.7.4-pg16 # Start fresh @@ -456,10 +456,10 @@ docker-compose up -d --build ```bash # Check health status -docker inspect --format='{{.State.Health.Status}}' dhamps-vdb +docker inspect --format='{{.State.Health.Status}}' embapi # View health check logs -docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' dhamps-vdb +docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' embapi ``` ### Docker build fails with network/TLS errors @@ -468,13 +468,13 @@ If you encounter errors like "TLS: unspecified error" or "Permission denied" whe ```bash # Try building with host network mode -docker build --network=host -t dhamps-vdb:latest . +docker build --network=host -t embapi:latest . # Or use a different mirror -docker build --build-arg ALPINE_MIRROR=http://dl-2.alpinelinux.org/alpine/ -t dhamps-vdb:latest . +docker build --build-arg ALPINE_MIRROR=http://dl-2.alpinelinux.org/alpine/ -t embapi:latest . # Or pull a pre-built image if available -docker pull /dhamps-vdb:latest +docker pull /embapi:latest ``` These errors typically indicate network restrictions or firewall issues in the build environment. The Docker files are correct and will work in standard environments. @@ -486,7 +486,7 @@ These errors typically indicate network restrictions or firewall issues in the b ```bash docker build \ --build-arg GO_VERSION=1.24 \ - -t dhamps-vdb:custom . + -t embapi:custom . ``` ### Resource Limits @@ -495,7 +495,7 @@ Add to docker-compose.yml: ```yaml services: - dhamps-vdb: + embapi: deploy: resources: limits: @@ -516,7 +516,7 @@ docker network create app-network ```yaml services: - dhamps-vdb: + embapi: networks: - app-network @@ -554,7 +554,7 @@ curl http://localhost:8880/ ```bash # Connect to PostgreSQL -docker-compose exec postgres psql -U postgres -d dhamps_vdb +docker-compose exec postgres psql -U postgres -d embapi # Check pgvector extension \dx @@ -582,7 +582,7 @@ curl -X POST http://localhost:8880/v1/users \ ### Common Issues -1. **Container won't start**: Check logs with `docker-compose logs dhamps-vdb` +1. **Container won't start**: Check logs with `docker-compose logs embapi` 2. **Can't connect to database**: Verify postgres is healthy with `docker-compose ps` 3. **API returns 401**: Check that SERVICE_ADMINKEY is set correctly 4. **Port already in use**: Change API_PORT or POSTGRES_PORT in .env file diff --git a/DOCUMENTATION_SUMMARY.md b/DOCUMENTATION_SUMMARY.md index 84c6f4d..18a440d 100644 --- a/DOCUMENTATION_SUMMARY.md +++ b/DOCUMENTATION_SUMMARY.md @@ -2,7 +2,7 @@ ## Summary -Created **50 markdown files** with **17,102 lines** of comprehensive Hugo documentation for dhamps-vdb. +Created **50 markdown files** with **17,102 lines** of comprehensive Hugo documentation for embapi. ## Directory Structure diff --git a/Dockerfile b/Dockerfile index 7bfdadf..feb93e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest && \ sqlc generate --no-remote # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o dhamps-vdb main.go +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o embapi main.go # Runtime stage FROM alpine:3.21 @@ -38,7 +38,7 @@ RUN addgroup -g 1000 appuser && \ WORKDIR /app # Copy binary from builder -COPY --from=builder /build/dhamps-vdb . +COPY --from=builder /build/embapi . # Copy migrations (needed for database schema management) COPY --from=builder /build/internal/database/migrations ./internal/database/migrations @@ -57,4 +57,4 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ CMD wget --no-verbose --tries=1 --spider http://localhost:8880/docs || exit 1 # Run the application -CMD ["./dhamps-vdb"] +CMD ["./embapi"] diff --git a/README.md b/README.md index 6a96dcc..b6a0e28 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# dhamps-vdb +# EmbAPI Vector Database for the DH at Max Planck Society initiative -[![Go Report Card](https://goreportcard.com/badge/github.com/mpilhlt/dhamps-vdb?style=flat-square)](https://goreportcard.com/report/github.com/mpilhlt/dhamps-vdb) [![Release](https://img.shields.io/github/v/release/mpilhlt/dhamps-vdb.svg?style=flat-square&include_prereleases)](https://github.com/mpilhlt/dhamps-vdb/releases/latest) +[![Go Report Card](https://goreportcard.com/badge/github.com/mpilhlt/embapi?style=flat-square)](https://goreportcard.com/report/github.com/mpilhlt/embapi) [![Release](https://img.shields.io/github/v/release/mpilhlt/embapi.svg?style=flat-square&include_prereleases)](https://github.com/mpilhlt/embapi/releases/latest) ## Introduction -dhamps-vdb is a PostgreSQL-backed vector database with pgvector support, providing a RESTful API for managing embeddings in Retrieval Augmented Generation (RAG) workflows. Store embeddings for text snippets with metadata, then find similar content using cosine similarity search. +EmbAPI is a PostgreSQL-backed vector database with pgvector support, providing a RESTful API for managing embeddings in Retrieval Augmented Generation (RAG) workflows. Store embeddings for text snippets with metadata, then find similar content using cosine similarity search. The typical use case is as a RAG component: Create embeddings for your text collection, upload them with identifiers and optional metadata, then query for similar texts either by identifier (`GET`) or by posting raw embeddings (`POST`). The service returns text identifiers with similarity scores for use in your application. @@ -58,8 +58,8 @@ curl -X POST http://localhost:8880/v1/users \ -H "Content-Type: application/json" \ -d '{"user_handle": "alice", "name": "Alice Smith"}' -# Response includes: {"vdb_key": "alice_abc123..."} -# ⚠️ Save the vdb_key! It cannot be recovered. +# Response includes: {"embapi_key": "alice_abc123..."} +# ⚠️ Save the embapi_key! It cannot be recovered. ``` ### 3. Create an LLM Service Instance @@ -67,7 +67,7 @@ curl -X POST http://localhost:8880/v1/users \ ```bash # Use a system-provided definition (openai-large, openai-small, etc.) curl -X PUT http://localhost:8880/v1/llm-instances/alice/my-openai \ - -H "Authorization: Bearer ALICE_VDB_KEY" \ + -H "Authorization: Bearer ALICE_EMBAPI_KEY" \ -H "Content-Type: application/json" \ -d '{ "definition_owner": "_system", @@ -80,7 +80,7 @@ curl -X PUT http://localhost:8880/v1/llm-instances/alice/my-openai \ ```bash curl -X POST http://localhost:8880/v1/projects/alice \ - -H "Authorization: Bearer ALICE_VDB_KEY" \ + -H "Authorization: Bearer ALICE_EMBAPI_KEY" \ -H "Content-Type: application/json" \ -d '{ "project_handle": "my-texts", @@ -94,7 +94,7 @@ curl -X POST http://localhost:8880/v1/projects/alice \ ```bash curl -X POST http://localhost:8880/v1/embeddings/alice/my-texts \ - -H "Authorization: Bearer ALICE_VDB_KEY" \ + -H "Authorization: Bearer ALICE_EMBAPI_KEY" \ -H "Content-Type: application/json" \ -d '{ "embeddings": [{ @@ -112,7 +112,7 @@ curl -X POST http://localhost:8880/v1/embeddings/alice/my-texts \ ```bash # Get documents similar to doc1 curl "http://localhost:8880/v1/similars/alice/my-texts/doc1?threshold=0.7&limit=5" \ - -H "Authorization: Bearer ALICE_VDB_KEY" + -H "Authorization: Bearer ALICE_EMBAPI_KEY" ``` ### 7. Filter by Metadata @@ -120,34 +120,34 @@ curl "http://localhost:8880/v1/similars/alice/my-texts/doc1?threshold=0.7&limit= ```bash # Exclude documents from the same author curl "http://localhost:8880/v1/similars/alice/my-texts/doc1?threshold=0.7&metadata_path=author&metadata_value=John%20Doe" \ - -H "Authorization: Bearer ALICE_VDB_KEY" + -H "Authorization: Bearer ALICE_EMBAPI_KEY" ``` ## Getting Started -📚 **[Read the Full Documentation](https://mpilhlt.github.io/dhamps-vdb/)** +📚 **[Read the Full Documentation](https://mpilhlt.github.io/embapi/)** -- **[Installation Guide](https://mpilhlt.github.io/dhamps-vdb/getting-started/installation/)** - Build from source -- **[Docker Guide](https://mpilhlt.github.io/dhamps-vdb/getting-started/docker/)** - Detailed Docker deployment (also see [DOCKER.md](./DOCKER.md)) -- **[Configuration](https://mpilhlt.github.io/dhamps-vdb/getting-started/configuration/)** - Environment variables and options -- **[Your First Project](https://mpilhlt.github.io/dhamps-vdb/getting-started/first-project/)** - Complete walkthrough -- **[API Reference](https://mpilhlt.github.io/dhamps-vdb/api/)** - Complete API documentation +- **[Installation Guide](https://mpilhlt.github.io/embapi/getting-started/installation/)** - Build from source +- **[Docker Guide](https://mpilhlt.github.io/embapi/getting-started/docker/)** - Detailed Docker deployment (also see [DOCKER.md](./DOCKER.md)) +- **[Configuration](https://mpilhlt.github.io/embapi/getting-started/configuration/)** - Environment variables and options +- **[Your First Project](https://mpilhlt.github.io/embapi/getting-started/first-project/)** - Complete walkthrough +- **[API Reference](https://mpilhlt.github.io/embapi/api/)** - Complete API documentation ## Key Concepts -- **[Architecture](https://mpilhlt.github.io/dhamps-vdb/concepts/architecture/)** - How dhamps-vdb works -- **[Users & Authentication](https://mpilhlt.github.io/dhamps-vdb/concepts/users-and-auth/)** - API keys and roles -- **[Projects](https://mpilhlt.github.io/dhamps-vdb/concepts/projects/)** - Organizing embeddings -- **[LLM Services](https://mpilhlt.github.io/dhamps-vdb/concepts/llm-services/)** - Definitions vs. instances -- **[Similarity Search](https://mpilhlt.github.io/dhamps-vdb/concepts/similarity-search/)** - How it works +- **[Architecture](https://mpilhlt.github.io/embapi/concepts/architecture/)** - How EmbAPI works +- **[Users & Authentication](https://mpilhlt.github.io/embapi/concepts/users-and-auth/)** - API keys and roles +- **[Projects](https://mpilhlt.github.io/embapi/concepts/projects/)** - Organizing embeddings +- **[LLM Services](https://mpilhlt.github.io/embapi/concepts/llm-services/)** - Definitions vs. instances +- **[Similarity Search](https://mpilhlt.github.io/embapi/concepts/similarity-search/)** - How it works ## Common Tasks -- **[RAG Workflow](https://mpilhlt.github.io/dhamps-vdb/guides/rag-workflow/)** - End-to-end example -- **[Project Sharing](https://mpilhlt.github.io/dhamps-vdb/guides/project-sharing/)** - Collaborate with others -- **[Metadata Filtering](https://mpilhlt.github.io/dhamps-vdb/guides/metadata-filtering/)** - Exclude documents in search -- **[Metadata Validation](https://mpilhlt.github.io/dhamps-vdb/guides/metadata-validation/)** - Enforce schemas -- **[Public Projects](https://mpilhlt.github.io/dhamps-vdb/guides/public-projects/)** - Unauthenticated access +- **[RAG Workflow](https://mpilhlt.github.io/embapi/guides/rag-workflow/)** - End-to-end example +- **[Project Sharing](https://mpilhlt.github.io/embapi/guides/project-sharing/)** - Collaborate with others +- **[Metadata Filtering](https://mpilhlt.github.io/embapi/guides/metadata-filtering/)** - Exclude documents in search +- **[Metadata Validation](https://mpilhlt.github.io/embapi/guides/metadata-validation/)** - Enforce schemas +- **[Public Projects](https://mpilhlt.github.io/embapi/guides/public-projects/)** - Unauthenticated access ## Development @@ -159,7 +159,7 @@ go get ./... sqlc generate --no-remote # Build -go build -o build/dhamps-vdb main.go +go build -o build/embapi main.go # Or run directly go run main.go @@ -178,11 +178,11 @@ export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock go test -v ./... ``` -For more details, see the **[Testing Guide](https://mpilhlt.github.io/dhamps-vdb/development/testing/)**. +For more details, see the **[Testing Guide](https://mpilhlt.github.io/embapi/development/testing/)**. ## Contributing -Contributions are welcome! Please see our **[Contributing Guide](https://mpilhlt.github.io/dhamps-vdb/development/contributing/)** for details. +Contributions are welcome! Please see our **[Contributing Guide](https://mpilhlt.github.io/embapi/development/contributing/)** for details. ## License @@ -190,6 +190,6 @@ This project is licensed under the terms specified in the [LICENSE](./LICENSE) f ## Support -- 📖 [Documentation](https://mpilhlt.github.io/dhamps-vdb/) -- 🐛 [Report Issues](https://github.com/mpilhlt/dhamps-vdb/issues) -- 💬 [GitHub Discussions](https://github.com/mpilhlt/dhamps-vdb/discussions) +- 📖 [Documentation](https://mpilhlt.github.io/embapi/) +- 🐛 [Report Issues](https://github.com/mpilhlt/embapi/issues) +- 💬 [GitHub Discussions](https://github.com/mpilhlt/embapi/discussions) diff --git a/api/openapi.yml b/api/openapi.yml index 1162155..a3b98ee 100644 --- a/api/openapi.yml +++ b/api/openapi.yml @@ -34,7 +34,7 @@ paths: type: string email: type: string - VDBKey: + EmbAPIKey: type: string responses: "200": @@ -107,7 +107,7 @@ paths: type: string email: type: string - VDBKey: + EmbAPIKey: type: string responses: "200": @@ -308,7 +308,7 @@ components: email: type: string format: email - VDBKey: + EmbAPIKey: type: string minLength: 32 maxLength: 32 @@ -321,7 +321,7 @@ components: required: - handle - email - - VDBKey + - EmbAPIKey project: type: object properties: diff --git a/docker-compose.external-db.yml b/docker-compose.external-db.yml index cf7394d..7b10ef2 100644 --- a/docker-compose.external-db.yml +++ b/docker-compose.external-db.yml @@ -1,6 +1,6 @@ -# docker-compose configuration for dhamps-vdb connecting to an external database +# docker-compose configuration for embapi connecting to an external database # -# This example shows how to run dhamps-vdb as a standalone container +# This example shows how to run embapi as a standalone container # that connects to an external PostgreSQL database with pgvector. # # Prerequisites: @@ -14,12 +14,12 @@ # 3. Run: docker-compose up -d services: - # dhamps-vdb middleware connecting to external database - dhamps-vdb: + # embapi middleware connecting to external database + embapi: build: context: . dockerfile: Dockerfile - container_name: dhamps-vdb + container_name: embapi environment: # Service configuration SERVICE_DEBUG: ${SERVICE_DEBUG:-false} diff --git a/docker-compose.yml b/docker-compose.yml index 4c86236..d7c2c08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,11 @@ services: # PostgreSQL with pgvector extension postgres: image: pgvector/pgvector:0.7.4-pg16 - container_name: dhamps-postgres + container_name: embapi-postgres environment: POSTGRES_USER: ${SERVICE_DBUSER:-postgres} POSTGRES_PASSWORD: ${SERVICE_DBPASSWORD:-postgres} - POSTGRES_DB: ${SERVICE_DBNAME:-dhamps_vdb} + POSTGRES_DB: ${SERVICE_DBNAME:-embapi} ports: - "${POSTGRES_PORT:-5432}:5432" volumes: @@ -17,14 +17,14 @@ services: timeout: 5s retries: 5 networks: - - dhamps-network + - embapi-network - # dhamps-vdb middleware - dhamps-vdb: + # embapi middleware + embapi: build: context: . dockerfile: Dockerfile - container_name: dhamps-vdb + container_name: embapi depends_on: postgres: condition: service_healthy @@ -39,7 +39,7 @@ services: SERVICE_DBPORT: ${SERVICE_DBPORT:-5432} SERVICE_DBUSER: ${SERVICE_DBUSER:-postgres} SERVICE_DBPASSWORD: ${SERVICE_DBPASSWORD:-postgres} - SERVICE_DBNAME: ${SERVICE_DBNAME:-dhamps_vdb} + SERVICE_DBNAME: ${SERVICE_DBNAME:-embapi} # Security configuration SERVICE_ADMINKEY: ${SERVICE_ADMINKEY} @@ -47,12 +47,12 @@ services: ports: - "${API_PORT:-8880}:8880" networks: - - dhamps-network + - embapi-network restart: unless-stopped volumes: postgres_data: networks: - dhamps-network: + embapi-network: driver: bridge diff --git a/docker-setup.sh b/docker-setup.sh index 3881455..984208b 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Quick setup script for dhamps-vdb Docker deployment +# Quick setup script for embapi Docker deployment # # This script helps you quickly set up environment variables for Docker deployment. # @@ -8,7 +8,7 @@ set -e echo "=====================================" -echo "dhamps-vdb Docker Quick Setup" +echo "embapi Docker Quick Setup" echo "=====================================" echo "" diff --git a/docs/config.toml b/docs/config.toml index 6b5d75c..129d6d8 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -1,9 +1,9 @@ -baseURL = 'https://mpilhlt.github.io/dhamps-vdb/' +baseURL = 'https://mpilhlt.github.io/embapi/' languageCode = 'en-us' -title = 'dhamps-vdb Documentation' +title = 'EmbAPI Documentation' theme = 'book' [params] - BookRepo = 'https://github.com/mpilhlt/dhamps-vdb' + BookRepo = 'https://github.com/mpilhlt/embapi' BookEditPath = 'edit/main' BookSearch = true diff --git a/docs/content/_index.md b/docs/content/_index.md index 62e336a..b415090 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -1,15 +1,15 @@ --- -title: dhamps-vdb Documentation +title: embapi Documentation type: docs --- -# dhamps-vdb Documentation +# embapi Documentation -Welcome to the documentation for **dhamps-vdb**, a vector database designed for Digital Humanities applications at the Max Planck Society initiative. +Welcome to the documentation for **embapi**, a vector database designed for Digital Humanities applications at the Max Planck Society initiative. -## What is dhamps-vdb? +## What is embapi? -dhamps-vdb is a PostgreSQL-backed vector database with pgvector support, providing a RESTful API for managing embeddings in Retrieval Augmented Generation (RAG) workflows. It offers multi-user support, project management, and flexible embedding configurations. +embapi is a PostgreSQL-backed vector database with pgvector support, providing a RESTful API for managing embeddings in Retrieval Augmented Generation (RAG) workflows. It offers multi-user support, project management, and flexible embedding configurations. ## Key Features @@ -24,15 +24,15 @@ dhamps-vdb is a PostgreSQL-backed vector database with pgvector support, providi ## Quick Links - [Getting Started](/getting-started/) - Installation and first steps -- [Concepts](/concepts/) - Understand how dhamps-vdb works +- [Concepts](/concepts/) - Understand how embapi works - [API Reference](/api/) - Complete API documentation - [Guides](/guides/) - How-to guides for common tasks ## Getting Help - 📖 Browse this documentation -- 🐛 [Report issues](https://github.com/mpilhlt/dhamps-vdb/issues) -- 💬 [GitHub Discussions](https://github.com/mpilhlt/dhamps-vdb/discussions) +- 🐛 [Report issues](https://github.com/mpilhlt/embapi/issues) +- 💬 [GitHub Discussions](https://github.com/mpilhlt/embapi/discussions) ## Quick Example diff --git a/docs/content/api/_index.md b/docs/content/api/_index.md index 5930d11..b8cb5f8 100644 --- a/docs/content/api/_index.md +++ b/docs/content/api/_index.md @@ -5,7 +5,7 @@ weight: 4 # API Reference -Complete reference for the dhamps-vdb REST API. +Complete reference for the embapi REST API. ## API Version diff --git a/docs/content/api/authentication.md b/docs/content/api/authentication.md index 074a9fa..4e15f32 100644 --- a/docs/content/api/authentication.md +++ b/docs/content/api/authentication.md @@ -53,7 +53,7 @@ Set the admin key via environment variable when launching the service: ```bash export ADMIN_KEY="your-secure-admin-key-here" -./dhamps-vdb +./embapi ``` ### User Key diff --git a/docs/content/api/endpoints/_index.md b/docs/content/api/endpoints/_index.md index f7a9d92..34dba36 100644 --- a/docs/content/api/endpoints/_index.md +++ b/docs/content/api/endpoints/_index.md @@ -5,7 +5,7 @@ weight: 1 # API Endpoints -Complete reference for all dhamps-vdb API endpoints. +Complete reference for all embapi API endpoints. ## Endpoint Categories diff --git a/docs/content/api/endpoints/api-standards.md b/docs/content/api/endpoints/api-standards.md index e45c079..f7a7bfc 100644 --- a/docs/content/api/endpoints/api-standards.md +++ b/docs/content/api/endpoints/api-standards.md @@ -16,7 +16,7 @@ API standards are referenced by LLM service instances to determine how to authen - **Google Gemini**: API key as query parameter - **Ollama**: No authentication required -Pre-seeded standards are available for common providers. See [testdata/valid_api_standard_*.json](https://github.com/mpilhlt/dhamps-vdb/tree/main/testdata) for examples. +Pre-seeded standards are available for common providers. See [testdata/valid_api_standard_*.json](https://github.com/mpilhlt/embapi/tree/main/testdata) for examples. --- @@ -454,4 +454,4 @@ curl -X PUT "https://api.example.com/v1/llm-services/alice/my-vllm" \ - [LLM Services](llm-services/) - LLM service instances reference API standards - [Authentication](../authentication/) - User authentication methods -- [Testdata Examples](https://github.com/mpilhlt/dhamps-vdb/tree/main/testdata) - Example API standard definitions +- [Testdata Examples](https://github.com/mpilhlt/embapi/tree/main/testdata) - Example API standard definitions diff --git a/docs/content/concepts/_index.md b/docs/content/concepts/_index.md index 9d49ae0..00b9474 100644 --- a/docs/content/concepts/_index.md +++ b/docs/content/concepts/_index.md @@ -5,11 +5,11 @@ weight: 2 # Core Concepts -Understanding the key concepts behind dhamps-vdb helps you make the most of its features. This section explains the fundamental building blocks and how they work together. +Understanding the key concepts behind embapi helps you make the most of its features. This section explains the fundamental building blocks and how they work together. ## Overview -dhamps-vdb is a vector database designed for Retrieval Augmented Generation (RAG) workflows. It stores embeddings with metadata and provides fast similarity search capabilities. +embapi is a vector database designed for Retrieval Augmented Generation (RAG) workflows. It stores embeddings with metadata and provides fast similarity search capabilities. ## Key Components @@ -21,4 +21,4 @@ dhamps-vdb is a vector database designed for Retrieval Augmented Generation (RAG ## Architecture -dhamps-vdb uses PostgreSQL with the pgvector extension for vector operations. It provides a RESTful API with token-based authentication and supports multi-user environments with project sharing. +embapi uses PostgreSQL with the pgvector extension for vector operations. It provides a RESTful API with token-based authentication and supports multi-user environments with project sharing. diff --git a/docs/content/concepts/architecture.md b/docs/content/concepts/architecture.md index 1151663..e4f399b 100644 --- a/docs/content/concepts/architecture.md +++ b/docs/content/concepts/architecture.md @@ -5,7 +5,7 @@ weight: 1 # Architecture -dhamps-vdb is a vector database API designed for RAG (Retrieval Augmented Generation) workflows in Digital Humanities research. +embapi is a vector database API designed for RAG (Retrieval Augmented Generation) workflows in Digital Humanities research. ## System Overview @@ -17,7 +17,7 @@ dhamps-vdb is a vector database API designed for RAG (Retrieval Augmented Genera │ HTTP/REST │ ┌──────▼──────────────────────────┐ -│ dhamps-vdb API Server │ +│ embapi API Server │ │ ┌──────────────────────────┐ │ │ │ Authentication Layer │ │ │ └────────┬─────────────────┘ │ diff --git a/docs/content/concepts/embeddings.md b/docs/content/concepts/embeddings.md index a2e057d..20b88f0 100644 --- a/docs/content/concepts/embeddings.md +++ b/docs/content/concepts/embeddings.md @@ -5,7 +5,7 @@ weight: 4 # Embeddings -Embeddings are vector representations of text stored in dhamps-vdb for similarity search and retrieval. +Embeddings are vector representations of text stored in embapi for similarity search and retrieval. ## What are Embeddings? diff --git a/docs/content/concepts/llm-services.md b/docs/content/concepts/llm-services.md index d1c74b9..a9f238e 100644 --- a/docs/content/concepts/llm-services.md +++ b/docs/content/concepts/llm-services.md @@ -9,7 +9,7 @@ LLM Services configure embedding generation, defining models, dimensions, and AP ## Architecture -dhamps-vdb separates LLM services into two concepts: +embapi separates LLM services into two concepts: ### LLM Service Definitions @@ -48,7 +48,7 @@ The `_system` user provides default definitions: ```bash GET /v1/llm-services/_system -Authorization: Bearer user_vdb_key +Authorization: Bearer user_embapi_key ``` Returns list of available system definitions. @@ -159,7 +159,7 @@ List all accessible instances (owned + shared): ```bash GET /v1/llm-services/alice -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Returns instances where alice is owner or has been granted access. @@ -168,7 +168,7 @@ Returns instances where alice is owner or has been granted access. ```bash GET /v1/llm-services/alice/my-openai -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Returns instance configuration (API key not included). @@ -190,7 +190,7 @@ Only owner can update instances. ```bash DELETE /v1/llm-services/alice/my-openai -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` **Constraints:** @@ -310,8 +310,8 @@ Planned features: Users must generate embeddings externally: 1. Generate embeddings using LLM API (OpenAI, Cohere, etc.) -2. Upload pre-generated embeddings to dhamps-vdb -3. Use dhamps-vdb for storage and similarity search +2. Upload pre-generated embeddings to embapi +3. Use embapi for storage and similarity search ## Common Patterns diff --git a/docs/content/concepts/projects.md b/docs/content/concepts/projects.md index 18fecb2..4e1ced2 100644 --- a/docs/content/concepts/projects.md +++ b/docs/content/concepts/projects.md @@ -244,7 +244,7 @@ List all projects owned by a user: ```bash GET /v1/projects/alice -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Returns array of project objects. @@ -253,7 +253,7 @@ Returns array of project objects. ```bash GET /v1/projects/alice/research -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Returns full project object including: @@ -290,7 +290,7 @@ PUT /v1/projects/alice/research ```bash DELETE /v1/projects/alice/research -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` **Cascading deletion:** diff --git a/docs/content/concepts/similarity-search.md b/docs/content/concepts/similarity-search.md index 6e13155..43be063 100644 --- a/docs/content/concepts/similarity-search.md +++ b/docs/content/concepts/similarity-search.md @@ -130,7 +130,7 @@ Value to **exclude** from results. ### Cosine Similarity -dhamps-vdb uses cosine similarity: +embapi uses cosine similarity: ``` similarity = 1 - cosine_distance diff --git a/docs/content/concepts/users-and-auth.md b/docs/content/concepts/users-and-auth.md index c1c7c1f..f164b6c 100644 --- a/docs/content/concepts/users-and-auth.md +++ b/docs/content/concepts/users-and-auth.md @@ -5,7 +5,7 @@ weight: 2 # Users and Authentication -dhamps-vdb uses token-based authentication with API keys for all operations. +embapi uses token-based authentication with API keys for all operations. ## User Model @@ -14,7 +14,7 @@ dhamps-vdb uses token-based authentication with API keys for all operations. - **user_handle**: Unique identifier (3-20 characters, alphanumeric + underscore) - **name**: Full name (optional) - **email**: Email address (unique, required) -- **vdb_key**: API key (SHA-256 hash, 64 characters) +- **embapi_key**: API key (SHA-256 hash, 64 characters) - **created_at**: Timestamp of creation - **updated_at**: Timestamp of last update @@ -82,12 +82,12 @@ Authorization: Bearer ADMIN_KEY "user_handle": "researcher1", "name": "Research User", "email": "researcher@example.com", - "vdb_key": "024v2013621509245f2e24abcdef...", + "embapi_key": "024v2013621509245f2e24abcdef...", "created_at": "2024-01-15T10:30:00Z" } ``` -**Important:** Save the `vdb_key` immediately - it cannot be recovered later. +**Important:** Save the `embapi_key` immediately - it cannot be recovered later. ### User Handle Restrictions @@ -104,7 +104,7 @@ Users can view their own information: ```bash GET /v1/users/alice -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Admins can view any user: @@ -131,7 +131,7 @@ Users can update their own information: ```bash PATCH /v1/users/alice -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key { "name": "Alice Smith-Jones", @@ -145,7 +145,7 @@ Users can delete their own account: ```bash DELETE /v1/users/alice -Authorization: Bearer alice_vdb_key +Authorization: Bearer alice_embapi_key ``` Admins can delete any user: @@ -218,9 +218,9 @@ Users own three types of resources: ### User Key vs. LLM API Keys -dhamps-vdb handles two types of keys: +embapi handles two types of keys: -1. **User API keys** (`vdb_key`): Authenticate users to dhamps-vdb +1. **User API keys** (`embapi_key`): Authenticate users to embapi - Stored as SHA-256 hash in database - Never encrypted (one-way hash) - Returned only once on user creation @@ -286,8 +286,8 @@ curl -X POST http://localhost:8880/v1/users \ "name": "Research User" }' -# Save returned vdb_key -export USER_KEY="returned-vdb-key" +# Save returned embapi_key +export USER_KEY="returned-embapi-key" # User verifies access curl -X GET http://localhost:8880/v1/users/researcher1 \ diff --git a/docs/content/deployment/_index.md b/docs/content/deployment/_index.md index f423702..2f58957 100644 --- a/docs/content/deployment/_index.md +++ b/docs/content/deployment/_index.md @@ -5,11 +5,11 @@ weight: 5 # Deployment Guide -Deploy dhamps-vdb to production environments. +Deploy embapi to production environments. ## Deployment Options -dhamps-vdb can be deployed in several ways: +embapi can be deployed in several ways: - **Docker Compose** - Simplest option, includes PostgreSQL - **Docker with External Database** - Production-ready setup diff --git a/docs/content/deployment/database.md b/docs/content/deployment/database.md index ba9be71..e0b1953 100644 --- a/docs/content/deployment/database.md +++ b/docs/content/deployment/database.md @@ -5,7 +5,7 @@ weight: 2 # Database Setup -dhamps-vdb requires PostgreSQL 11 or later with the pgvector extension. +embapi requires PostgreSQL 11 or later with the pgvector extension. ## Requirements @@ -91,16 +91,16 @@ Create the database: ```sql -- Create database -CREATE DATABASE dhamps_vdb; +CREATE DATABASE embapi; ``` ### Step 2: Create User -Create a dedicated user for dhamps-vdb: +Create a dedicated user for embapi: ```sql -- Create user with password -CREATE USER dhamps_user WITH PASSWORD 'secure_password_here'; +CREATE USER embapi_user WITH PASSWORD 'secure_password_here'; ``` **Best practices:** @@ -114,21 +114,21 @@ Grant necessary permissions: ```sql -- Grant database privileges -GRANT ALL PRIVILEGES ON DATABASE dhamps_vdb TO dhamps_user; +GRANT ALL PRIVILEGES ON DATABASE embapi TO embapi_user; -- Connect to the database -\c dhamps_vdb +\c embapi -- Grant schema privileges -GRANT ALL ON SCHEMA public TO dhamps_user; +GRANT ALL ON SCHEMA public TO embapi_user; -- For PostgreSQL 15+, also grant: -GRANT CREATE ON DATABASE dhamps_vdb TO dhamps_user; +GRANT CREATE ON DATABASE embapi TO embapi_user; ``` ### Step 4: Enable pgvector Extension -Still connected to the `dhamps_vdb` database: +Still connected to the `embapi` database: ```sql -- Enable pgvector extension @@ -171,14 +171,14 @@ Exit psql: ## Connection String Format -dhamps-vdb connects using these environment variables: +embapi connects using these environment variables: ```bash SERVICE_DBHOST=localhost # Database hostname SERVICE_DBPORT=5432 # Database port -SERVICE_DBUSER=dhamps_user # Database username +SERVICE_DBUSER=embapi_user # Database username SERVICE_DBPASSWORD=password # Database password -SERVICE_DBNAME=dhamps_vdb # Database name +SERVICE_DBNAME=embapi # Database name ``` The connection string format used internally: @@ -235,7 +235,7 @@ sudo apt install pgbouncer # Configure /etc/pgbouncer/pgbouncer.ini [databases] -dhamps_vdb = host=localhost port=5432 dbname=dhamps_vdb +embapi = host=localhost port=5432 dbname=embapi [pgbouncer] listen_addr = 127.0.0.1 @@ -247,7 +247,7 @@ max_client_conn = 1000 default_pool_size = 20 ``` -Connect dhamps-vdb to PgBouncer instead of PostgreSQL directly. +Connect embapi to PgBouncer instead of PostgreSQL directly. ### SSL/TLS Encryption @@ -263,7 +263,7 @@ ssl_ca_file = '/etc/postgresql/ca.crt' Update connection string: ```bash -# Update dhamps-vdb configuration +# Update embapi configuration SERVICE_DBHOST=db.example.com # Change connection to use SSL (requires code modification or PostgreSQL parameter) ``` @@ -275,8 +275,8 @@ Restrict access in `pg_hba.conf`: ``` # TYPE DATABASE USER ADDRESS METHOD -# Allow dhamps_user from application server only -host dhamps_vdb dhamps_user 10.0.1.0/24 md5 +# Allow embapi_user from application server only +host embapi embapi_user 10.0.1.0/24 md5 # Allow localhost connections local all all peer @@ -298,17 +298,17 @@ Daily backup script: ```bash #!/bin/bash -# /usr/local/bin/backup-dhamps-vdb.sh +# /usr/local/bin/backup-embapi.sh -BACKUP_DIR="/backups/dhamps-vdb" +BACKUP_DIR="/backups/embapi" DATE=$(date +%Y%m%d_%H%M%S) -DB_NAME="dhamps_vdb" +DB_NAME="embapi" # Create backup -pg_dump -U dhamps_user -h localhost $DB_NAME | gzip > "$BACKUP_DIR/dhamps-vdb-$DATE.sql.gz" +pg_dump -U embapi_user -h localhost $DB_NAME | gzip > "$BACKUP_DIR/embapi-$DATE.sql.gz" # Keep last 30 days -find $BACKUP_DIR -name "dhamps-vdb-*.sql.gz" -mtime +30 -delete +find $BACKUP_DIR -name "embapi-*.sql.gz" -mtime +30 -delete # Verify backup if [ $? -eq 0 ]; then @@ -323,15 +323,15 @@ Set up cron job: ```bash # Run daily at 2 AM -0 2 * * * /usr/local/bin/backup-dhamps-vdb.sh +0 2 * * * /usr/local/bin/backup-embapi.sh ``` ### Restore from Backup ```bash # Decompress and restore -gunzip -c /backups/dhamps-vdb/dhamps-vdb-20240208.sql.gz | \ - psql -U dhamps_user -h localhost dhamps_vdb +gunzip -c /backups/embapi/embapi-20240208.sql.gz | \ + psql -U embapi_user -h localhost embapi ``` ### Point-in-Time Recovery (PITR) @@ -350,7 +350,7 @@ archive_command = 'test ! -f /archive/%f && cp %p /archive/%f' ```sql -- Database size -SELECT pg_size_pretty(pg_database_size('dhamps_vdb')); +SELECT pg_size_pretty(pg_database_size('embapi')); -- Table sizes SELECT @@ -366,7 +366,7 @@ ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC; ```sql -- Active connections -SELECT count(*) FROM pg_stat_activity WHERE datname = 'dhamps_vdb'; +SELECT count(*) FROM pg_stat_activity WHERE datname = 'embapi'; -- Connection details SELECT @@ -377,7 +377,7 @@ SELECT state, query FROM pg_stat_activity -WHERE datname = 'dhamps_vdb'; +WHERE datname = 'embapi'; ``` ### Monitor Performance @@ -411,7 +411,7 @@ sudo systemctl status postgresql sudo tail -f /var/log/postgresql/postgresql-16-main.log # Test connection -psql -h localhost -U dhamps_user -d dhamps_vdb +psql -h localhost -U embapi_user -d embapi ``` ### pgvector Extension Not Found @@ -428,17 +428,17 @@ SELECT * FROM pg_available_extensions WHERE name = 'vector'; ```sql -- Reconnect as superuser -\c dhamps_vdb postgres +\c embapi postgres -- Re-grant privileges -GRANT ALL PRIVILEGES ON DATABASE dhamps_vdb TO dhamps_user; -GRANT ALL ON SCHEMA public TO dhamps_user; -GRANT ALL ON ALL TABLES IN SCHEMA public TO dhamps_user; -GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO dhamps_user; +GRANT ALL PRIVILEGES ON DATABASE embapi TO embapi_user; +GRANT ALL ON SCHEMA public TO embapi_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO embapi_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO embapi_user; -- For future objects -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO dhamps_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO dhamps_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO embapi_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO embapi_user; ``` ### Out of Disk Space @@ -477,12 +477,12 @@ VACUUM ANALYZE; ## Migration from Other Databases -dhamps-vdb is designed specifically for PostgreSQL with pgvector. Migration from other databases requires: +embapi is designed specifically for PostgreSQL with pgvector. Migration from other databases requires: 1. Export data from source database 2. Set up PostgreSQL with pgvector -3. Transform data to match dhamps-vdb schema -4. Import using dhamps-vdb API or direct SQL +3. Transform data to match embapi schema +4. Import using embapi API or direct SQL ## Further Reading diff --git a/docs/content/deployment/docker.md b/docs/content/deployment/docker.md index 7c7ea6b..e3ba924 100644 --- a/docs/content/deployment/docker.md +++ b/docs/content/deployment/docker.md @@ -5,7 +5,7 @@ weight: 1 # Docker Deployment -This guide covers production-focused Docker deployment for dhamps-vdb. +This guide covers production-focused Docker deployment for embapi. ## Overview @@ -23,8 +23,8 @@ For detailed Docker setup instructions, see [Getting Started with Docker](../../ ```bash # Clone repository -git clone https://github.com/mpilhlt/dhamps-vdb.git -cd dhamps-vdb +git clone https://github.com/mpilhlt/embapi.git +cd embapi # Generate secure keys ./docker-setup.sh @@ -50,7 +50,7 @@ Always run behind a reverse proxy (nginx, Traefik, Caddy) for: Example nginx configuration: ```nginx -upstream dhamps-vdb { +upstream embapi { server localhost:8880; } @@ -62,7 +62,7 @@ server { ssl_certificate_key /path/to/key.pem; location / { - proxy_pass http://dhamps-vdb; + proxy_pass http://embapi; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -77,7 +77,7 @@ Set resource limits in `docker-compose.yml`: ```yaml services: - dhamps-vdb: + embapi: deploy: resources: limits: @@ -107,8 +107,8 @@ services: postgres: image: pgvector/pgvector:0.7.4-pg16 # Specific version - dhamps-vdb: - image: dhamps-vdb:v0.1.0 # Tag your builds + embapi: + image: embapi:v0.1.0 # Tag your builds ``` ### Health Monitoring @@ -117,10 +117,10 @@ The image includes health checks. Monitor with: ```bash # Check health status -docker inspect --format='{{.State.Health.Status}}' dhamps-vdb +docker inspect --format='{{.State.Health.Status}}' embapi # View health check logs -docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' dhamps-vdb +docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' embapi # Integrate with monitoring (Prometheus, etc.) ``` @@ -131,7 +131,7 @@ Configure logging drivers in `docker-compose.yml`: ```yaml services: - dhamps-vdb: + embapi: logging: driver: "json-file" options: @@ -143,7 +143,7 @@ Or use centralized logging: ```yaml services: - dhamps-vdb: + embapi: logging: driver: "syslog" options: @@ -169,16 +169,16 @@ Update `.env`: ```bash SERVICE_DBHOST=db.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi ``` Modify `docker-compose.yml` to remove postgres service: ```yaml services: - dhamps-vdb: + embapi: build: . ports: - "8880:8880" @@ -194,7 +194,7 @@ Run multiple instances behind a load balancer: ```yaml services: - dhamps-vdb: + embapi: build: . deploy: replicas: 3 @@ -218,17 +218,17 @@ For high availability: ```bash # Automated daily backups -0 2 * * * docker-compose exec -T postgres pg_dump -U postgres dhamps_vdb | gzip > /backups/dhamps-vdb-$(date +\%Y\%m\%d).sql.gz +0 2 * * * docker-compose exec -T postgres pg_dump -U postgres embapi | gzip > /backups/embapi-$(date +\%Y\%m\%d).sql.gz # Keep last 30 days -find /backups -name "dhamps-vdb-*.sql.gz" -mtime +30 -delete +find /backups -name "embapi-*.sql.gz" -mtime +30 -delete ``` ### Volume Backups ```bash # Backup Docker volume -docker run --rm -v dhamps-vdb_postgres_data:/data -v /backups:/backup alpine tar czf /backup/postgres-data-$(date +\%Y\%m\%d).tar.gz /data +docker run --rm -v embapi_postgres_data:/data -v /backups:/backup alpine tar czf /backup/postgres-data-$(date +\%Y\%m\%d).tar.gz /data ``` ### Environment Configuration Backups @@ -247,23 +247,23 @@ gpg --encrypt --recipient admin@example.com .env > .env.gpg docker stats # Check container logs -docker-compose logs -f --tail=100 dhamps-vdb +docker-compose logs -f --tail=100 embapi # Check slow queries (if database is slow) -docker-compose exec postgres psql -U postgres -d dhamps_vdb -c "SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10;" +docker-compose exec postgres psql -U postgres -d embapi -c "SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10;" ``` ### Network Connectivity Issues ```bash # Test database connection from container -docker-compose exec dhamps-vdb nc -zv postgres 5432 +docker-compose exec embapi nc -zv postgres 5432 # Check DNS resolution -docker-compose exec dhamps-vdb nslookup postgres +docker-compose exec embapi nslookup postgres # Test API from inside container -docker-compose exec dhamps-vdb wget -O- http://localhost:8880/docs +docker-compose exec embapi wget -O- http://localhost:8880/docs ``` ### Update and Rollback diff --git a/docs/content/deployment/environment-variables.md b/docs/content/deployment/environment-variables.md index 6e143f0..459ee5d 100644 --- a/docs/content/deployment/environment-variables.md +++ b/docs/content/deployment/environment-variables.md @@ -5,11 +5,11 @@ weight: 3 # Environment Variables -Complete reference for all environment variables used by dhamps-vdb. +Complete reference for all environment variables used by embapi. ## Overview -dhamps-vdb is configured entirely through environment variables. These can be set: +embapi is configured entirely through environment variables. These can be set: 1. **In a `.env` file** (recommended for Docker) 2. **As system environment variables** @@ -17,7 +17,7 @@ dhamps-vdb is configured entirely through environment variables. These can be se ## Required Variables -These variables **must** be set for dhamps-vdb to function: +These variables **must** be set for embapi to function: ### SERVICE_ADMINKEY @@ -216,7 +216,7 @@ Database username. ```bash SERVICE_DBUSER=postgres # Default superuser -SERVICE_DBUSER=dhamps_user # Dedicated user (recommended) +SERVICE_DBUSER=embapi_user # Dedicated user (recommended) ``` **Security:** @@ -264,8 +264,8 @@ Database name. **Example:** ```bash -SERVICE_DBNAME=dhamps_vdb # Production database -SERVICE_DBNAME=dhamps_test # Testing database +SERVICE_DBNAME=embapi # Production database +SERVICE_DBNAME=embapi_test # Testing database ``` ## Configuration Examples @@ -281,7 +281,7 @@ SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=postgres -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi SERVICE_ADMINKEY=dev-admin-key-not-for-production ENCRYPTION_KEY=dev-encryption-key-min-32-chars-long ``` @@ -297,7 +297,7 @@ SERVICE_DBHOST=postgres SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=secure_db_password_here -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi SERVICE_ADMINKEY=generated_admin_key_from_setup_script ENCRYPTION_KEY=generated_encryption_key_from_setup_script ``` @@ -311,9 +311,9 @@ SERVICE_HOST=0.0.0.0 SERVICE_PORT=8880 SERVICE_DBHOST=db.internal.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_prod_user +SERVICE_DBUSER=embapi_prod_user SERVICE_DBPASSWORD= -SERVICE_DBNAME=dhamps_vdb_prod +SERVICE_DBNAME=embapi_prod SERVICE_ADMINKEY= ENCRYPTION_KEY= ``` @@ -359,15 +359,15 @@ docker run -d \ -e SERVICE_ADMINKEY=admin-key \ -e ENCRYPTION_KEY=encryption-key \ -e SERVICE_DBHOST=db-host \ - dhamps-vdb:latest + embapi:latest ``` With docker-compose.yml: ```yaml services: - dhamps-vdb: - image: dhamps-vdb:latest + embapi: + image: embapi:latest environment: SERVICE_DEBUG: "false" SERVICE_HOST: "0.0.0.0" @@ -382,7 +382,7 @@ services: Some variables support command-line flags: ```bash -./dhamps-vdb \ +./embapi \ --debug \ --host 0.0.0.0 \ --port 8880 \ @@ -391,7 +391,7 @@ Some variables support command-line flags: --db-port 5432 \ --db-user postgres \ --db-password password \ - --db-name dhamps_vdb + --db-name embapi ``` **Note:** ENCRYPTION_KEY must be set as environment variable, not flag. @@ -440,7 +440,7 @@ Error: failed to connect to database ```bash # Start service with debug logging -SERVICE_DEBUG=true ./dhamps-vdb +SERVICE_DEBUG=true ./embapi # Check if service starts successfully curl http://localhost:8880/docs diff --git a/docs/content/deployment/security.md b/docs/content/deployment/security.md index 7de673a..7c62549 100644 --- a/docs/content/deployment/security.md +++ b/docs/content/deployment/security.md @@ -5,11 +5,11 @@ weight: 4 # Security Best Practices -Comprehensive guide for securing your dhamps-vdb production deployment. +Comprehensive guide for securing your embapi production deployment. ## Security Overview -dhamps-vdb handles sensitive data including embeddings, metadata, and API credentials. This guide covers essential security measures for production deployments. +embapi handles sensitive data including embeddings, metadata, and API credentials. This guide covers essential security measures for production deployments. ## HTTPS/TLS Configuration @@ -21,7 +21,7 @@ dhamps-vdb handles sensitive data including embeddings, metadata, and API creden ### Using a Reverse Proxy -**Never expose dhamps-vdb directly to the internet.** Always use a reverse proxy with TLS termination. +**Never expose embapi directly to the internet.** Always use a reverse proxy with TLS termination. #### Nginx with Let's Encrypt @@ -31,10 +31,10 @@ Install Certbot: sudo apt install nginx certbot python3-certbot-nginx ``` -Configure nginx (`/etc/nginx/sites-available/dhamps-vdb`): +Configure nginx (`/etc/nginx/sites-available/embapi`): ```nginx -upstream dhamps_backend { +upstream embapi_backend { server 127.0.0.1:8880; keepalive 32; } @@ -73,7 +73,7 @@ server { # Proxy settings location / { - proxy_pass http://dhamps_backend; + proxy_pass http://embapi_backend; proxy_http_version 1.1; proxy_set_header Host $host; @@ -102,7 +102,7 @@ server { Enable site and get certificate: ```bash -sudo ln -s /etc/nginx/sites-available/dhamps-vdb /etc/nginx/sites-enabled/ +sudo ln -s /etc/nginx/sites-available/embapi /etc/nginx/sites-enabled/ sudo certbot --nginx -d api.example.com sudo systemctl reload nginx ``` @@ -141,18 +141,18 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./letsencrypt:/letsencrypt - dhamps-vdb: + embapi: build: . labels: - "traefik.enable=true" - - "traefik.http.routers.dhamps-vdb.rule=Host(`api.example.com`)" - - "traefik.http.routers.dhamps-vdb.entrypoints=websecure" - - "traefik.http.routers.dhamps-vdb.tls.certresolver=letsencrypt" + - "traefik.http.routers.embapi.rule=Host(`api.example.com`)" + - "traefik.http.routers.embapi.entrypoints=websecure" + - "traefik.http.routers.embapi.tls.certresolver=letsencrypt" # Redirect HTTP to HTTPS - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" - - "traefik.http.routers.dhamps-vdb-http.rule=Host(`api.example.com`)" - - "traefik.http.routers.dhamps-vdb-http.entrypoints=web" - - "traefik.http.routers.dhamps-vdb-http.middlewares=redirect-to-https" + - "traefik.http.routers.embapi-http.rule=Host(`api.example.com`)" + - "traefik.http.routers.embapi-http.entrypoints=web" + - "traefik.http.routers.embapi-http.middlewares=redirect-to-https" ``` #### Caddy (Automatic HTTPS) @@ -313,7 +313,7 @@ Document and test recovery procedure: ```sql -- In pg_hba.conf # Allow only from application server - host dhamps_vdb dhamps_user 10.0.1.0/24 md5 + host embapi embapi_user 10.0.1.0/24 md5 ``` 2. **Firewall rules:** @@ -339,8 +339,8 @@ Document and test recovery procedure: 2. **Dedicated user:** ```sql -- Not superuser - CREATE USER dhamps_user WITH PASSWORD 'secure_password'; - GRANT ALL PRIVILEGES ON DATABASE dhamps_vdb TO dhamps_user; + CREATE USER embapi_user WITH PASSWORD 'secure_password'; + GRANT ALL PRIVILEGES ON DATABASE embapi TO embapi_user; ``` 3. **SSL/TLS connections:** @@ -362,7 +362,7 @@ Document and test recovery procedure: 2. **Backup encryption:** ```bash # Encrypt backup - pg_dump -U postgres dhamps_vdb | gzip | \ + pg_dump -U postgres embapi | gzip | \ gpg --encrypt --recipient admin@example.com > backup.sql.gz.gpg ``` @@ -417,7 +417,7 @@ Internet ↓ [Reverse Proxy] ← (DMZ/Public subnet) ↓ -[dhamps-vdb] ← (Application subnet) +[embapi] ← (Application subnet) ↓ [PostgreSQL] ← (Database subnet) ``` @@ -439,7 +439,7 @@ server { location /v1/ { limit_req zone=api_limit burst=20 nodelay; limit_req_status 429; - proxy_pass http://dhamps_backend; + proxy_pass http://embapi_backend; } } ``` @@ -456,13 +456,13 @@ server { 1. **Automated daily backups:** ```bash #!/bin/bash - # /usr/local/bin/backup-dhamps.sh + # /usr/local/bin/backup-embapi.sh DATE=$(date +%Y%m%d_%H%M%S) - BACKUP_DIR="/backups/dhamps-vdb" + BACKUP_DIR="/backups/embapi" # Create backup - pg_dump -U dhamps_user dhamps_vdb | gzip > \ + pg_dump -U embapi_user embapi | gzip > \ "$BACKUP_DIR/db-$DATE.sql.gz" # Encrypt backup @@ -477,7 +477,7 @@ server { # Upload to offsite storage aws s3 cp "$BACKUP_DIR/db-$DATE.sql.gz.gpg" \ - s3://backups/dhamps-vdb/ --storage-class GLACIER + s3://backups/embapi/ --storage-class GLACIER # Cleanup old local backups (keep 7 days) find $BACKUP_DIR -name "db-*.sql.gz.gpg" -mtime +7 -delete @@ -485,7 +485,7 @@ server { 2. **Cron schedule:** ```cron - 0 2 * * * /usr/local/bin/backup-dhamps.sh + 0 2 * * * /usr/local/bin/backup-embapi.sh ``` 3. **Test restores regularly:** diff --git a/docs/content/development/_index.md b/docs/content/development/_index.md index 367e09a..b7c8772 100644 --- a/docs/content/development/_index.md +++ b/docs/content/development/_index.md @@ -5,7 +5,7 @@ weight: 6 # Development Guide -Information for developers contributing to dhamps-vdb. +Information for developers contributing to embapi. ## Getting Started with Development @@ -20,7 +20,7 @@ This section covers: ## Project Structure ``` -dhamps-vdb/ +embapi/ ├── main.go # Application entry point ├── internal/ │ ├── auth/ # Authentication logic @@ -36,7 +36,7 @@ dhamps-vdb/ 1. Make changes to code 2. Generate sqlc code if database queries changed: `sqlc generate` 3. Run tests: `go test -v ./...` -4. Build: `go build -o dhamps-vdb main.go` +4. Build: `go build -o embapi main.go` 5. Submit pull request ## Resources diff --git a/docs/content/development/architecture.md b/docs/content/development/architecture.md index f4ed2bb..ed7cefc 100644 --- a/docs/content/development/architecture.md +++ b/docs/content/development/architecture.md @@ -5,12 +5,12 @@ weight: 3 # Technical Architecture -This document provides a technical deep-dive into dhamps-vdb's architecture for developers who want to understand or modify the codebase. +This document provides a technical deep-dive into embapi's architecture for developers who want to understand or modify the codebase. ## Project Structure ``` -dhamps-vdb/ +embapi/ ├── main.go # Application entry point ├── go.mod # Go module definition ├── go.sum # Dependency checksums @@ -111,7 +111,7 @@ func main() { // 4. Create HTTP router and Huma API router := http.NewServeMux() - api := humago.New(router, huma.DefaultConfig("dhamps-vdb", "0.1.0")) + api := humago.New(router, huma.DefaultConfig("embapi", "0.1.0")) // 5. Register all routes handlers.AddRoutes(pool, keyGen, api) @@ -291,7 +291,7 @@ Migration files are numbered sequentially: CREATE TABLE users ( user_id SERIAL PRIMARY KEY, user_handle TEXT UNIQUE NOT NULL, - vdb_key TEXT UNIQUE NOT NULL, + embapi_key TEXT UNIQUE NOT NULL, ... ); @@ -315,7 +315,7 @@ Write SQL, generate type-safe Go code: ```sql -- name: UpsertUser :one INSERT INTO users ( - user_handle, name, email, vdb_key, created_at, updated_at + user_handle, name, email, embapi_key, created_at, updated_at ) VALUES ( $1, $2, $3, $4, NOW(), NOW() ) @@ -500,7 +500,7 @@ func Decrypt(ciphertext []byte, key []byte) ([]byte, error) { ## Huma Framework Integration -dhamps-vdb uses [Huma](https://huma.rocks/) for API development: +embapi uses [Huma](https://huma.rocks/) for API development: ### Benefits @@ -658,8 +658,8 @@ if err != nil { Go's `internal/` directory enforces package privacy: ```go -// This import works within dhamps-vdb -import "github.com/mpilhlt/dhamps-vdb/internal/database" +// This import works within embapi +import "github.com/mpilhlt/embapi/internal/database" // This import would FAIL from external projects // Enforced by Go compiler @@ -766,13 +766,13 @@ See [Testing Guide](../testing/) for comprehensive testing documentation. ```bash # Development build -go build -o dhamps-vdb main.go +go build -o embapi main.go # Production build with optimizations -go build -ldflags="-s -w" -o dhamps-vdb main.go +go build -ldflags="-s -w" -o embapi main.go # Cross-compilation -GOOS=linux GOARCH=amd64 go build -o dhamps-vdb-linux main.go +GOOS=linux GOARCH=amd64 go build -o embapi-linux main.go ``` ### Docker Build @@ -786,15 +786,15 @@ WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go build -ldflags="-s -w" -o dhamps-vdb main.go +RUN go build -ldflags="-s -w" -o embapi main.go # Runtime stage FROM alpine:latest RUN apk --no-cache add ca-certificates WORKDIR /root/ -COPY --from=builder /app/dhamps-vdb . +COPY --from=builder /app/embapi . EXPOSE 8880 -CMD ["./dhamps-vdb"] +CMD ["./embapi"] ``` ### Configuration diff --git a/docs/content/development/contributing.md b/docs/content/development/contributing.md index 43b778a..959466c 100644 --- a/docs/content/development/contributing.md +++ b/docs/content/development/contributing.md @@ -5,7 +5,7 @@ weight: 2 # Contributing Guide -Thank you for your interest in contributing to dhamps-vdb! This guide will help you get started. +Thank you for your interest in contributing to embapi! This guide will help you get started. ## Development Setup @@ -21,8 +21,8 @@ Thank you for your interest in contributing to dhamps-vdb! This guide will help ```bash # Clone repository -git clone https://github.com/mpilhlt/dhamps-vdb.git -cd dhamps-vdb +git clone https://github.com/mpilhlt/embapi.git +cd embapi # Install dependencies go get ./... @@ -31,7 +31,7 @@ go get ./... sqlc generate --no-remote # Build application -go build -o build/dhamps-vdb main.go +go build -o build/embapi main.go # Or run directly go run main.go @@ -56,7 +56,7 @@ SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=password -SERVICE_DBNAME=dhamps_vdb_dev +SERVICE_DBNAME=embapi_dev # Encryption key (32+ characters) ENCRYPTION_KEY=your-secure-encryption-key-min-32-chars @@ -72,7 +72,7 @@ For local development: # Start PostgreSQL with pgvector podman run -p 5432:5432 \ -e POSTGRES_PASSWORD=password \ - -e POSTGRES_DB=dhamps_vdb_dev \ + -e POSTGRES_DB=embapi_dev \ pgvector/pgvector:0.7.4-pg16 # Or use docker-compose @@ -99,7 +99,7 @@ go test -v ./... ### Go Formatting -dhamps-vdb follows standard Go conventions: +embapi follows standard Go conventions: ```bash # Format all code @@ -130,8 +130,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" // Internal packages - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" ) // Exported function with doc comment @@ -518,7 +518,7 @@ What should happen What actually happens ## Environment -- dhamps-vdb version: v0.1.0 +- embapi version: v0.1.0 - Go version: 1.21.5 - PostgreSQL version: 16.1 - OS: Ubuntu 22.04 @@ -680,4 +680,4 @@ Contributors are acknowledged in: - GitHub contributors page - Release notes (for significant contributions) -Thank you for contributing to dhamps-vdb! 🎉 +Thank you for contributing to embapi! 🎉 diff --git a/docs/content/development/performance.md b/docs/content/development/performance.md index b9071ab..b8ec63b 100644 --- a/docs/content/development/performance.md +++ b/docs/content/development/performance.md @@ -5,7 +5,7 @@ weight: 4 # Performance Optimization Guide -This guide covers performance optimization strategies for dhamps-vdb, including query optimization, indexing, caching, and performance testing. +This guide covers performance optimization strategies for embapi, including query optimization, indexing, caching, and performance testing. ## Query Optimization diff --git a/docs/content/development/testing.md b/docs/content/development/testing.md index b59372d..b8c0875 100644 --- a/docs/content/development/testing.md +++ b/docs/content/development/testing.md @@ -5,11 +5,11 @@ weight: 1 # Testing Guide -This guide covers how to run and write tests for dhamps-vdb. +This guide covers how to run and write tests for embapi. ## Running Tests -dhamps-vdb uses integration tests that spin up real PostgreSQL containers using [testcontainers](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/). This approach ensures tests run against actual database instances with pgvector support. +embapi uses integration tests that spin up real PostgreSQL containers using [testcontainers](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/). This approach ensures tests run against actual database instances with pgvector support. ### Prerequisites @@ -124,7 +124,7 @@ import ( "context" "testing" - "github.com/mpilhlt/dhamps-vdb/internal/database" + "github.com/mpilhlt/embapi/internal/database" "github.com/stretchr/testify/assert" ) diff --git a/docs/content/getting-started/_index.md b/docs/content/getting-started/_index.md index 2b30682..83886f7 100644 --- a/docs/content/getting-started/_index.md +++ b/docs/content/getting-started/_index.md @@ -3,13 +3,13 @@ title: "Getting Started" weight: 1 --- -# Getting Started with dhamps-vdb +# Getting Started with embapi -This section helps you get dhamps-vdb up and running quickly. Whether you're using Docker or compiling from source, you'll find everything you need to start using the vector database API. +This section helps you get embapi up and running quickly. Whether you're using Docker or compiling from source, you'll find everything you need to start using the vector database API. ## What You'll Learn -- How to install and run dhamps-vdb +- How to install and run embapi - How to configure the service for your environment - Basic usage patterns and workflows - Creating your first project and embeddings @@ -24,7 +24,7 @@ Before you begin, ensure you have: ## Quick Links -- [Installation](installation/) - Compile and install dhamps-vdb +- [Installation](installation/) - Compile and install embapi - [Docker Deployment](docker/) - Run with Docker (recommended) - [Configuration](configuration/) - Environment variables and options - [Quick Start](quick-start/) - Your first API requests diff --git a/docs/content/getting-started/configuration.md b/docs/content/getting-started/configuration.md index 8997b24..a1d307d 100644 --- a/docs/content/getting-started/configuration.md +++ b/docs/content/getting-started/configuration.md @@ -5,7 +5,7 @@ weight: 2 # Configuration -Configure dhamps-vdb using environment variables or command-line options. +Configure embapi using environment variables or command-line options. ## Environment Variables @@ -49,9 +49,9 @@ SERVICE_PORT=8880 # Database Configuration SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi # Security SERVICE_ADMINKEY=your-secure-admin-key-here @@ -63,14 +63,14 @@ ENCRYPTION_KEY=your-32-character-encryption-key-minimum You can also provide configuration via command-line flags: ```bash -./dhamps-vdb \ +./embapi \ --debug \ -p 8880 \ --db-host localhost \ --db-port 5432 \ - --db-user dhamps_user \ + --db-user embapi_user \ --db-password secure_password \ - --db-name dhamps_vdb \ + --db-name embapi \ --admin-key your-admin-key ``` @@ -122,7 +122,7 @@ SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=password -SERVICE_DBNAME=dhamps_vdb_dev +SERVICE_DBNAME=embapi_dev SERVICE_ADMINKEY=dev-admin-key-change-me ENCRYPTION_KEY=dev-encryption-key-32-chars-min ``` @@ -136,9 +136,9 @@ SERVICE_HOST=0.0.0.0 SERVICE_PORT=8880 SERVICE_DBHOST=prod-db.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_prod +SERVICE_DBUSER=embapi_prod SERVICE_DBPASSWORD=$(cat /run/secrets/db_password) -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi SERVICE_ADMINKEY=$(cat /run/secrets/admin_key) ENCRYPTION_KEY=$(cat /run/secrets/encryption_key) ``` diff --git a/docs/content/getting-started/docker.md b/docs/content/getting-started/docker.md index 82504d8..ee1f5d2 100644 --- a/docs/content/getting-started/docker.md +++ b/docs/content/getting-started/docker.md @@ -5,16 +5,16 @@ weight: 2 # Docker Deployment -Deploy dhamps-vdb using Docker containers. This is the recommended approach for most users. +Deploy embapi using Docker containers. This is the recommended approach for most users. ## Quick Start -The fastest way to get dhamps-vdb running with Docker: +The fastest way to get embapi running with Docker: ```bash # Clone the repository -git clone https://github.com/mpilhlt/dhamps-vdb.git -cd dhamps-vdb +git clone https://github.com/mpilhlt/embapi.git +cd embapi # Run automated setup (generates secure keys) ./docker-setup.sh @@ -23,7 +23,7 @@ cd dhamps-vdb docker-compose up -d # Check logs -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi # Access the API curl http://localhost:8880/docs @@ -33,7 +33,7 @@ curl http://localhost:8880/docs The Docker Compose setup includes: -- **dhamps-vdb**: The vector database API service +- **embapi**: The vector database API service - **PostgreSQL 16**: Database with pgvector extension - **Persistent storage**: Named volume for database data @@ -77,7 +77,7 @@ services: image: pgvector/pgvector:0.7.4-pg16 # PostgreSQL with pgvector support - dhamps-vdb: + embapi: build: . # The API service depends_on: @@ -107,24 +107,24 @@ docker-compose up -d ### Option 2: Standalone Container with External Database -Run only the dhamps-vdb container: +Run only the embapi container: ```bash # Build the image -docker build -t dhamps-vdb:latest . +docker build -t embapi:latest . # Run the container docker run -d \ - --name dhamps-vdb \ + --name embapi \ -p 8880:8880 \ -e SERVICE_DBHOST=your-db-host \ -e SERVICE_DBPORT=5432 \ -e SERVICE_DBUSER=dbuser \ -e SERVICE_DBPASSWORD=dbpass \ - -e SERVICE_DBNAME=dhamps_vdb \ + -e SERVICE_DBNAME=embapi \ -e SERVICE_ADMINKEY=admin-key \ -e ENCRYPTION_KEY=encryption-key \ - dhamps-vdb:latest + embapi:latest ``` **Use when:** @@ -138,7 +138,7 @@ Modify `docker-compose.yml` to remove the postgres service: ```yaml services: - dhamps-vdb: + embapi: build: . ports: - "${API_PORT:-8880}:8880" @@ -152,19 +152,19 @@ services: ### Standard Build ```bash -docker build -t dhamps-vdb:latest . +docker build -t embapi:latest . ``` ### Custom Tag ```bash -docker build -t dhamps-vdb:v0.1.0 . +docker build -t embapi:v0.1.0 . ``` ### Clean Build (No Cache) ```bash -docker build --no-cache -t dhamps-vdb:latest . +docker build --no-cache -t embapi:latest . ``` ### Multi-Stage Build Details @@ -201,7 +201,7 @@ docker-compose logs docker-compose logs -f # Specific service -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi docker-compose logs -f postgres ``` @@ -225,7 +225,7 @@ docker-compose down -v docker-compose restart # Restart specific service -docker-compose restart dhamps-vdb +docker-compose restart embapi ``` ## Data Persistence @@ -250,17 +250,17 @@ docker volume ls ### Inspect Volume ```bash -docker volume inspect dhamps-vdb_postgres_data +docker volume inspect embapi_postgres_data ``` ### Backup Database ```bash # Create backup -docker-compose exec postgres pg_dump -U postgres dhamps_vdb > backup.sql +docker-compose exec postgres pg_dump -U postgres embapi > backup.sql # Restore from backup -docker-compose exec -T postgres psql -U postgres dhamps_vdb < backup.sql +docker-compose exec -T postgres psql -U postgres embapi < backup.sql ``` ## Networking @@ -278,7 +278,7 @@ http://localhost:8880 Use the service name as hostname: ``` -http://dhamps-vdb:8880 +http://embapi:8880 ``` ### Custom Network @@ -348,7 +348,7 @@ curl http://localhost:8880/docs ```bash # Connect to PostgreSQL -docker-compose exec postgres psql -U postgres -d dhamps_vdb +docker-compose exec postgres psql -U postgres -d embapi # Check pgvector extension \dx @@ -374,7 +374,7 @@ curl -X POST http://localhost:8880/v1/users \ Check logs: ```bash -docker-compose logs dhamps-vdb +docker-compose logs embapi ``` Common issues: @@ -392,7 +392,7 @@ docker-compose ps docker-compose logs postgres # Test connection -docker-compose exec postgres psql -U postgres -d dhamps_vdb -c "SELECT 1;" +docker-compose exec postgres psql -U postgres -d embapi -c "SELECT 1;" ``` ### Can't Connect to API @@ -402,10 +402,10 @@ docker-compose exec postgres psql -U postgres -d dhamps_vdb -c "SELECT 1;" docker ps # Check port mapping -docker port dhamps-vdb +docker port embapi # Test from inside container -docker-compose exec dhamps-vdb wget -O- http://localhost:8880/docs +docker-compose exec embapi wget -O- http://localhost:8880/docs # Test from host curl http://localhost:8880/docs @@ -417,7 +417,7 @@ The container runs as non-root user `appuser` (UID 1000). If you have permission ```bash # Check volume permissions -docker volume inspect dhamps-vdb_postgres_data +docker volume inspect embapi_postgres_data ``` ### Reset Everything @@ -427,7 +427,7 @@ docker volume inspect dhamps-vdb_postgres_data docker-compose down -v # Remove images -docker rmi dhamps-vdb:latest +docker rmi embapi:latest docker rmi pgvector/pgvector:0.7.4-pg16 # Start fresh @@ -440,7 +440,7 @@ If Docker build fails with network errors: ```bash # Try with host network -docker build --network=host -t dhamps-vdb:latest . +docker build --network=host -t embapi:latest . ``` ## Advanced Configuration @@ -451,7 +451,7 @@ Add to `docker-compose.yml`: ```yaml services: - dhamps-vdb: + embapi: deploy: resources: limits: @@ -474,7 +474,7 @@ HEALTHCHECK --interval=30s --timeout=3s \ View health status: ```bash -docker inspect --format='{{.State.Health.Status}}' dhamps-vdb +docker inspect --format='{{.State.Health.Status}}' embapi ``` ### Custom Dockerfile Builds @@ -484,7 +484,7 @@ You can customize the build: ```bash docker build \ --build-arg GO_VERSION=1.24 \ - -t dhamps-vdb:custom . + -t embapi:custom . ``` ## External Database Setup @@ -495,37 +495,37 @@ If using an external PostgreSQL database: ```sql -- Create database -CREATE DATABASE dhamps_vdb; +CREATE DATABASE embapi; -- Create user -CREATE USER dhamps_user WITH PASSWORD 'secure_password'; +CREATE USER embapi_user WITH PASSWORD 'secure_password'; -- Grant privileges -GRANT ALL PRIVILEGES ON DATABASE dhamps_vdb TO dhamps_user; +GRANT ALL PRIVILEGES ON DATABASE embapi TO embapi_user; -- Connect to database -\c dhamps_vdb +\c embapi -- Grant schema permissions -GRANT ALL ON SCHEMA public TO dhamps_user; +GRANT ALL ON SCHEMA public TO embapi_user; -- Enable pgvector CREATE EXTENSION IF NOT EXISTS vector; ``` -### Configure dhamps-vdb +### Configure embapi Update `.env`: ```bash SERVICE_DBHOST=external-db.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi ``` -Then run only the dhamps-vdb service or use a standalone container. +Then run only the embapi service or use a standalone container. ## Next Steps diff --git a/docs/content/getting-started/first-project.md b/docs/content/getting-started/first-project.md index a29c571..7367fe6 100644 --- a/docs/content/getting-started/first-project.md +++ b/docs/content/getting-started/first-project.md @@ -5,7 +5,7 @@ weight: 4 # First Project -Step-by-step guide to creating your first complete project in dhamps-vdb. +Step-by-step guide to creating your first complete project in embapi. ## Overview @@ -35,10 +35,10 @@ curl -X POST http://localhost:8880/v1/users \ }' ``` -Save the returned `vdb_key` to a variable: +Save the returned `embapi_key` to a variable: ```bash -export USER_KEY="your-returned-vdb-key" +export USER_KEY="your-returned-embapi-key" ``` ### Verify Authentication @@ -338,7 +338,7 @@ curl -X DELETE http://localhost:8880/v1/embeddings/researcher1/literature-analys ```bash #!/bin/bash -USER_KEY="your-vdb-key" +USER_KEY="your-embapi-key" PROJECT="researcher1/literature-analysis" API_URL="http://localhost:8880" diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index ae9bcb1..339d142 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -5,7 +5,7 @@ weight: 1 # Installation -Install dhamps-vdb by compiling from source. +Install embapi by compiling from source. ## Prerequisites @@ -17,15 +17,15 @@ Install dhamps-vdb by compiling from source. ```bash # Clone the repository -git clone https://github.com/mpilhlt/dhamps-vdb.git -cd dhamps-vdb +git clone https://github.com/mpilhlt/embapi.git +cd embapi # Install dependencies and generate code go get ./... sqlc generate --no-remote # Build the binary -go build -o build/dhamps-vdb main.go +go build -o build/embapi main.go ``` ## Detailed Steps @@ -53,10 +53,10 @@ This creates Go code from SQL queries in `internal/database/queries/`. Compile the application: ```bash -go build -o build/dhamps-vdb main.go +go build -o build/embapi main.go ``` -The binary will be created at `build/dhamps-vdb`. +The binary will be created at `build/embapi`. ## Running Without Building @@ -73,7 +73,7 @@ This is useful during development but slower than running a pre-built binary. Check that the binary was created successfully: ```bash -./build/dhamps-vdb --help +./build/embapi --help ``` You should see the available command-line options. @@ -116,7 +116,7 @@ Clean the build cache if you encounter issues: ```bash go clean -cache -go build -o build/dhamps-vdb main.go +go build -o build/embapi main.go ``` ### Missing Dependencies diff --git a/docs/content/getting-started/quick-start.md b/docs/content/getting-started/quick-start.md index 5237db1..9c993cd 100644 --- a/docs/content/getting-started/quick-start.md +++ b/docs/content/getting-started/quick-start.md @@ -9,7 +9,7 @@ Complete walkthrough from installation to searching similar documents using curl ## Prerequisites -- dhamps-vdb installed and running +- embapi installed and running - Admin API key configured - PostgreSQL with pgvector ready @@ -35,12 +35,12 @@ curl -X POST http://localhost:8880/v1/users \ "user_handle": "alice", "name": "Alice Smith", "email": "alice@example.com", - "vdb_key": "024v2013621509245f2e24...", + "embapi_key": "024v2013621509245f2e24...", "created_at": "2024-01-15T10:30:00Z" } ``` -**Save the `vdb_key`** - it cannot be recovered later. +**Save the `embapi_key`** - it cannot be recovered later. ## 2. Create an LLM Service Instance @@ -48,7 +48,7 @@ Create an LLM service configuration: ```bash curl -X PUT http://localhost:8880/v1/llm-services/alice/my-openai \ - -H "Authorization: Bearer alice_vdb_key" \ + -H "Authorization: Bearer alice_embapi_key" \ -H "Content-Type: application/json" \ -d '{ "endpoint": "https://api.openai.com/v1/embeddings", @@ -79,7 +79,7 @@ Create a project to organize your embeddings: ```bash curl -X POST http://localhost:8880/v1/projects/alice \ - -H "Authorization: Bearer alice_vdb_key" \ + -H "Authorization: Bearer alice_embapi_key" \ -H "Content-Type: application/json" \ -d '{ "project_handle": "research-docs", @@ -108,7 +108,7 @@ Upload document embeddings to your project: ```bash curl -X POST http://localhost:8880/v1/embeddings/alice/research-docs \ - -H "Authorization: Bearer alice_vdb_key" \ + -H "Authorization: Bearer alice_embapi_key" \ -H "Content-Type: application/json" \ -d '{ "embeddings": [ @@ -156,7 +156,7 @@ Find documents similar to an already-stored document: ```bash curl -X GET "http://localhost:8880/v1/similars/alice/research-docs/doc1?count=5&threshold=0.7" \ - -H "Authorization: Bearer alice_vdb_key" + -H "Authorization: Bearer alice_embapi_key" ``` **Response:** @@ -184,7 +184,7 @@ Search without storing the query embedding: ```bash curl -X POST "http://localhost:8880/v1/similars/alice/research-docs?count=5&threshold=0.7" \ - -H "Authorization: Bearer alice_vdb_key" \ + -H "Authorization: Bearer alice_embapi_key" \ -H "Content-Type: application/json" \ -d '{ "vector": [0.12, 0.22, 0.32, ..., 0.52] @@ -197,7 +197,7 @@ Exclude documents from a specific author when searching: ```bash curl -X GET "http://localhost:8880/v1/similars/alice/research-docs/doc1?count=5&metadata_path=author&metadata_value=Alice" \ - -H "Authorization: Bearer alice_vdb_key" + -H "Authorization: Bearer alice_embapi_key" ``` This excludes all documents where `metadata.author` equals "Alice". @@ -208,14 +208,14 @@ Get all embeddings in your project: ```bash curl -X GET "http://localhost:8880/v1/embeddings/alice/research-docs?limit=10&offset=0" \ - -H "Authorization: Bearer alice_vdb_key" + -H "Authorization: Bearer alice_embapi_key" ``` Get a specific embedding: ```bash curl -X GET http://localhost:8880/v1/embeddings/alice/research-docs/doc1 \ - -H "Authorization: Bearer alice_vdb_key" + -H "Authorization: Bearer alice_embapi_key" ``` ## Complete Workflow Example @@ -235,7 +235,7 @@ USER_RESPONSE=$(curl -s -X POST "$API_URL/v1/users" \ -H "Content-Type: application/json" \ -d '{"user_handle":"alice","name":"Alice Smith","email":"alice@example.com"}') -USER_KEY=$(echo $USER_RESPONSE | jq -r '.vdb_key') +USER_KEY=$(echo $USER_RESPONSE | jq -r '.embapi_key') echo "User created with key: $USER_KEY" # 2. Create LLM service instance diff --git a/docs/content/guides/_index.md b/docs/content/guides/_index.md index 9b042cf..5836825 100644 --- a/docs/content/guides/_index.md +++ b/docs/content/guides/_index.md @@ -5,11 +5,11 @@ weight: 3 # User Guides -Step-by-step guides for common tasks and workflows with dhamps-vdb. +Step-by-step guides for common tasks and workflows with embapi. ## Available Guides -This section contains practical guides for using dhamps-vdb in real-world scenarios: +This section contains practical guides for using embapi in real-world scenarios: - **RAG Workflows** - Implement Retrieval Augmented Generation - **Project Sharing** - Collaborate with other users diff --git a/docs/content/guides/batch-operations.md b/docs/content/guides/batch-operations.md index aaac38c..cfdc283 100644 --- a/docs/content/guides/batch-operations.md +++ b/docs/content/guides/batch-operations.md @@ -5,7 +5,7 @@ weight: 7 # Batch Operations Guide -This guide explains how to efficiently upload multiple embeddings, manage large datasets, and implement best practices for batch operations in dhamps-vdb. +This guide explains how to efficiently upload multiple embeddings, manage large datasets, and implement best practices for batch operations in embapi. ## Overview diff --git a/docs/content/guides/metadata-filtering.md b/docs/content/guides/metadata-filtering.md index 3a27b48..c3ca016 100644 --- a/docs/content/guides/metadata-filtering.md +++ b/docs/content/guides/metadata-filtering.md @@ -14,7 +14,7 @@ When searching for similar documents, you may want to exclude results that share - Filter out documents from the same source when finding related content - Exclude documents with the same category when exploring diversity -dhamps-vdb provides metadata filtering using query parameters that perform **negative matching** - they exclude documents where the metadata field matches the specified value. +embapi provides metadata filtering using query parameters that perform **negative matching** - they exclude documents where the metadata field matches the specified value. ## Query Parameters diff --git a/docs/content/guides/metadata-validation.md b/docs/content/guides/metadata-validation.md index c345fd1..188bff5 100644 --- a/docs/content/guides/metadata-validation.md +++ b/docs/content/guides/metadata-validation.md @@ -9,7 +9,7 @@ This guide explains how to use JSON Schema validation to ensure consistent metad ## Overview -dhamps-vdb supports optional metadata validation using JSON Schema. When you define a metadata schema for a project, the API automatically validates all embedding metadata against that schema, ensuring data quality and consistency. +embapi supports optional metadata validation using JSON Schema. When you define a metadata schema for a project, the API automatically validates all embedding metadata against that schema, ensuring data quality and consistency. Benefits: - Enforce consistent metadata structure across all embeddings diff --git a/docs/content/guides/ownership-transfer.md b/docs/content/guides/ownership-transfer.md index 8c47b2b..7471ce1 100644 --- a/docs/content/guides/ownership-transfer.md +++ b/docs/content/guides/ownership-transfer.md @@ -5,7 +5,7 @@ weight: 4 # Ownership Transfer Guide -This guide explains how to transfer project ownership between users in dhamps-vdb. +This guide explains how to transfer project ownership between users in embapi. ## Overview diff --git a/docs/content/guides/project-sharing.md b/docs/content/guides/project-sharing.md index b9276e1..eadb972 100644 --- a/docs/content/guides/project-sharing.md +++ b/docs/content/guides/project-sharing.md @@ -5,7 +5,7 @@ weight: 2 # Project Sharing Guide -This guide explains how to share projects with specific users for collaborative work in dhamps-vdb. +This guide explains how to share projects with specific users for collaborative work in embapi. ## Overview diff --git a/docs/content/guides/public-projects.md b/docs/content/guides/public-projects.md index c2854fe..88a4ade 100644 --- a/docs/content/guides/public-projects.md +++ b/docs/content/guides/public-projects.md @@ -269,7 +269,7 @@ Researchers worldwide can access the data without credentials, but only authoriz ### Public API Backend -Build a public search API on top of dhamps-vdb: +Build a public search API on top of embapi: ```python import requests diff --git a/docs/content/guides/rag-workflow.md b/docs/content/guides/rag-workflow.md index 30c5138..4b73eb0 100644 --- a/docs/content/guides/rag-workflow.md +++ b/docs/content/guides/rag-workflow.md @@ -5,20 +5,20 @@ weight: 1 # Complete RAG Workflow Guide -This guide demonstrates a complete Retrieval Augmented Generation (RAG) workflow using dhamps-vdb as your vector database. +This guide demonstrates a complete Retrieval Augmented Generation (RAG) workflow using embapi as your vector database. ## Overview A typical RAG workflow involves: 1. Generate embeddings from your text content (using an external LLM service) -2. Upload embeddings to dhamps-vdb +2. Upload embeddings to embapi 3. Search for similar documents based on a query 4. Retrieve the relevant context 5. Use the context with an LLM to generate responses ## Prerequisites -- Access to dhamps-vdb API with a valid API key +- Access to embapi API with a valid API key - An external LLM service for generating embeddings (e.g., OpenAI, Cohere) - Text content you want to process @@ -45,7 +45,7 @@ embedding_vector = response.data[0].embedding ## Step 2: Create LLM Service Instance -Before uploading embeddings, create an LLM service instance in dhamps-vdb that matches your embedding configuration: +Before uploading embeddings, create an LLM service instance in embapi that matches your embedding configuration: ```bash curl -X PUT "https://api.example.com/v1/llm-services/alice/my-openai" \ @@ -88,7 +88,7 @@ curl -X PUT "https://api.example.com/v1/projects/alice/my-documents" \ }' ``` -## Step 4: Upload Embeddings to dhamps-vdb +## Step 4: Upload Embeddings to embapi Upload your pre-generated embeddings along with metadata and optional text content: @@ -232,14 +232,14 @@ import requests # Configuration DHAMPS_API = "https://api.example.com" -DHAMPS_KEY = "your-dhamps-api-key" +EMBAPI_KEY = "your-embapi-key" OPENAI_KEY = "your-openai-key" # Initialize OpenAI client = openai.OpenAI(api_key=OPENAI_KEY) def embed_and_store(text_id, text, metadata=None): - """Generate embedding and store in dhamps-vdb""" + """Generate embedding and store in embapi""" # Generate embedding response = client.embeddings.create( model="text-embedding-3-large", @@ -248,11 +248,11 @@ def embed_and_store(text_id, text, metadata=None): ) vector = response.data[0].embedding - # Upload to dhamps-vdb + # Upload to embapi requests.post( f"{DHAMPS_API}/v1/embeddings/alice/my-documents", headers={ - "Authorization": f"Bearer {DHAMPS_KEY}", + "Authorization": f"Bearer {EMBAPI_KEY}", "Content-Type": "application/json" }, json={ @@ -277,11 +277,11 @@ def search_similar(query, count=5): ) query_vector = response.data[0].embedding - # Search in dhamps-vdb + # Search in embapi result = requests.post( f"{DHAMPS_API}/v1/similars/alice/my-documents?count={count}", headers={ - "Authorization": f"Bearer {DHAMPS_KEY}", + "Authorization": f"Bearer {EMBAPI_KEY}", "Content-Type": "application/json" }, json={"vector": query_vector} @@ -294,7 +294,7 @@ def retrieve_context(doc_ids): for doc_id in doc_ids: response = requests.get( f"{DHAMPS_API}/v1/embeddings/alice/my-documents/{doc_id}", - headers={"Authorization": f"Bearer {DHAMPS_KEY}"} + headers={"Authorization": f"Bearer {EMBAPI_KEY}"} ) docs.append(response.json()) return docs diff --git a/docs/content/reference/_index.md b/docs/content/reference/_index.md index 99baa22..1911bbf 100644 --- a/docs/content/reference/_index.md +++ b/docs/content/reference/_index.md @@ -17,4 +17,4 @@ Technical reference materials and specifications. - **OpenAPI Specification**: Available at `/openapi.yaml` on any running instance - **Go Package Documentation**: Coming soon -- **Source Code**: [github.com/mpilhlt/dhamps-vdb](https://github.com/mpilhlt/dhamps-vdb) +- **Source Code**: [github.com/mpilhlt/embapi](https://github.com/mpilhlt/embapi) diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index ef561a1..48d6078 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -5,11 +5,11 @@ weight: 1 # Configuration Reference -Complete reference for configuring dhamps-vdb. This guide consolidates all configuration options, including environment variables, command-line flags, and Docker configuration. +Complete reference for configuring embapi. This guide consolidates all configuration options, including environment variables, command-line flags, and Docker configuration. ## Overview -dhamps-vdb is configured through a combination of: +embapi is configured through a combination of: 1. **Environment variables** (recommended) 2. **Command-line flags** (overrides environment variables) @@ -64,7 +64,7 @@ Options for connecting to the PostgreSQL database with pgvector extension. - PostgreSQL 12+ (16+ recommended) - pgvector extension installed and enabled - User must have CREATE, ALTER, DROP, INSERT, SELECT, UPDATE, DELETE privileges -- Database must exist before starting dhamps-vdb +- Database must exist before starting embapi **Common Database Hosts:** - `localhost` - Local PostgreSQL instance @@ -101,7 +101,7 @@ Critical security settings for authentication and encryption. ### .env File -The recommended way to configure dhamps-vdb. Create a `.env` file in the project root: +The recommended way to configure embapi. Create a `.env` file in the project root: ```bash # Copy template @@ -122,9 +122,9 @@ SERVICE_PORT=8880 # Database Configuration SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password_here -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi # Security Configuration SERVICE_ADMINKEY=generated_admin_key_here @@ -173,13 +173,13 @@ services: environment: POSTGRES_USER: ${SERVICE_DBUSER:-postgres} POSTGRES_PASSWORD: ${SERVICE_DBPASSWORD:-postgres} - POSTGRES_DB: ${SERVICE_DBNAME:-dhamps_vdb} + POSTGRES_DB: ${SERVICE_DBNAME:-embapi} ports: - "${POSTGRES_PORT:-5432}:5432" volumes: - postgres_data:/var/lib/postgresql/data - dhamps-vdb: + embapi: build: context: . dockerfile: Dockerfile @@ -194,7 +194,7 @@ services: SERVICE_DBPORT: ${SERVICE_DBPORT:-5432} SERVICE_DBUSER: ${SERVICE_DBUSER:-postgres} SERVICE_DBPASSWORD: ${SERVICE_DBPASSWORD:-postgres} - SERVICE_DBNAME: ${SERVICE_DBNAME:-dhamps_vdb} + SERVICE_DBNAME: ${SERVICE_DBNAME:-embapi} SERVICE_ADMINKEY: ${SERVICE_ADMINKEY} ENCRYPTION_KEY: ${ENCRYPTION_KEY} ports: @@ -203,7 +203,7 @@ services: **Docker-Specific Variables:** - `POSTGRES_PORT` - External port for PostgreSQL (default: 5432) -- `API_PORT` - External port for dhamps-vdb API (default: 8880) +- `API_PORT` - External port for embapi API (default: 8880) ### Docker Setup Script @@ -217,7 +217,7 @@ Automated setup using `docker-setup.sh`: docker-compose up -d # View logs -docker-compose logs -f dhamps-vdb +docker-compose logs -f embapi ``` The script automatically: @@ -232,19 +232,19 @@ For standalone container deployment: ```bash docker run -d \ - --name dhamps-vdb \ + --name embapi \ -e SERVICE_DEBUG=false \ -e SERVICE_HOST=0.0.0.0 \ -e SERVICE_PORT=8880 \ -e SERVICE_DBHOST=db.example.com \ -e SERVICE_DBPORT=5432 \ - -e SERVICE_DBUSER=dhamps_user \ + -e SERVICE_DBUSER=embapi_user \ -e SERVICE_DBPASSWORD=secure_password \ - -e SERVICE_DBNAME=dhamps_vdb \ + -e SERVICE_DBNAME=embapi \ -e SERVICE_ADMINKEY=admin_key_here \ -e ENCRYPTION_KEY=encryption_key_here \ -p 8880:8880 \ - dhamps-vdb:latest + embapi:latest ``` ### External Database @@ -255,9 +255,9 @@ Using `docker-compose.external-db.yml` for external PostgreSQL: # Set database connection in .env SERVICE_DBHOST=db.external.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=secure_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi # Start without bundled PostgreSQL docker-compose -f docker-compose.external-db.yml up -d @@ -278,14 +278,14 @@ SERVICE_DBHOST=localhost SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=postgres -SERVICE_DBNAME=dhamps_vdb_dev +SERVICE_DBNAME=embapi_dev SERVICE_ADMINKEY=dev-admin-key-not-for-production ENCRYPTION_KEY=dev-encryption-key-at-least-32-chars ``` **Start service:** ```bash -./dhamps-vdb +./embapi ``` ### Docker Development @@ -301,7 +301,7 @@ SERVICE_DBHOST=postgres SERVICE_DBPORT=5432 SERVICE_DBUSER=postgres SERVICE_DBPASSWORD=postgres -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi SERVICE_ADMINKEY=dev-admin-key ENCRYPTION_KEY=dev-encryption-32-chars-minimum ``` @@ -322,9 +322,9 @@ SERVICE_HOST=0.0.0.0 SERVICE_PORT=8880 SERVICE_DBHOST=prod-db.internal.example.com SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_prod_user +SERVICE_DBUSER=embapi_prod_user SERVICE_DBPASSWORD= -SERVICE_DBNAME=dhamps_vdb_prod +SERVICE_DBNAME=embapi_prod SERVICE_ADMINKEY= ENCRYPTION_KEY= ``` @@ -362,7 +362,7 @@ go test -v ./... ### Startup Validation -dhamps-vdb validates configuration on startup: +embapi validates configuration on startup: 1. **Required variables check** - Fails if missing 2. **Database connection test** - Verifies connectivity @@ -382,7 +382,7 @@ curl -X GET http://localhost:8880/v1/users \ -H "Authorization: Bearer ${SERVICE_ADMINKEY}" # Check database connectivity -docker-compose exec dhamps-vdb echo "Config OK" +docker-compose exec embapi echo "Config OK" ``` ### Common Issues @@ -458,9 +458,9 @@ SERVICE_HOST=localhost SERVICE_PORT=8880 SERVICE_DBHOST=192.168.1.100 SERVICE_DBPORT=5432 -SERVICE_DBUSER=dhamps_user +SERVICE_DBUSER=embapi_user SERVICE_DBPASSWORD=user_password -SERVICE_DBNAME=dhamps_vdb +SERVICE_DBNAME=embapi SERVICE_ADMINKEY=$(openssl rand -base64 32) ENCRYPTION_KEY=$(openssl rand -hex 32) ``` @@ -472,15 +472,15 @@ ConfigMap: apiVersion: v1 kind: ConfigMap metadata: - name: dhamps-vdb-config + name: embapi-config data: SERVICE_DEBUG: "false" SERVICE_HOST: "0.0.0.0" SERVICE_PORT: "8880" SERVICE_DBHOST: "postgres-service" SERVICE_DBPORT: "5432" - SERVICE_DBUSER: "dhamps_user" - SERVICE_DBNAME: "dhamps_vdb" + SERVICE_DBUSER: "embapi_user" + SERVICE_DBNAME: "embapi" ``` Secrets: @@ -488,7 +488,7 @@ Secrets: apiVersion: v1 kind: Secret metadata: - name: dhamps-vdb-secrets + name: embapi-secrets type: Opaque stringData: SERVICE_DBPASSWORD: "secure_db_password" @@ -500,18 +500,18 @@ stringData: ```bash # Create secrets -echo "admin_key_here" | docker secret create dhamps_admin_key - -echo "encryption_key" | docker secret create dhamps_encryption_key - +echo "admin_key_here" | docker secret create embapi_admin_key - +echo "encryption_key" | docker secret create embapi_encryption_key - # Reference in stack file services: - dhamps-vdb: + embapi: secrets: - - dhamps_admin_key - - dhamps_encryption_key + - embapi_admin_key + - embapi_encryption_key environment: - SERVICE_ADMINKEY_FILE: /run/secrets/dhamps_admin_key - ENCRYPTION_KEY_FILE: /run/secrets/dhamps_encryption_key + SERVICE_ADMINKEY_FILE: /run/secrets/embapi_admin_key + ENCRYPTION_KEY_FILE: /run/secrets/embapi_encryption_key ``` ## Related Documentation diff --git a/docs/content/reference/database-schema.md b/docs/content/reference/database-schema.md index 797917a..63f99d7 100644 --- a/docs/content/reference/database-schema.md +++ b/docs/content/reference/database-schema.md @@ -5,7 +5,7 @@ weight: 2 # Database Schema Reference -Complete reference for the dhamps-vdb PostgreSQL database schema. This document describes all tables, columns, types, constraints, relationships, and indexes. +Complete reference for the embapi PostgreSQL database schema. This document describes all tables, columns, types, constraints, relationships, and indexes. ## Overview @@ -39,14 +39,14 @@ Stores user accounts with API authentication. | `user_handle` | `VARCHAR(20)` | PRIMARY KEY | Unique user identifier (username) | | `name` | `TEXT` | | Full name or display name | | `email` | `TEXT` | UNIQUE, NOT NULL | Email address | -| `vdb_key` | `CHAR(64)` | UNIQUE, NOT NULL | API key (64-character hex) | +| `embapi_key` | `CHAR(64)` | UNIQUE, NOT NULL | API key (64-character hex) | | `created_at` | `TIMESTAMP` | NOT NULL | Creation timestamp | | `updated_at` | `TIMESTAMP` | NOT NULL | Last update timestamp | **Indexes:** - Primary key on `user_handle` - Unique constraint on `email` -- Unique constraint on `vdb_key` +- Unique constraint on `embapi_key` **Special Users:** - `_system` - Reserved system user for global LLM service definitions @@ -59,7 +59,7 @@ Stores user accounts with API authentication. - Can share LLM service instances (N:M via `instances_shared_with`) **Notes:** -- `vdb_key` is generated during user creation and returned once +- `embapi_key` is generated during user creation and returned once - Cannot be recovered if lost - Transmitted as `Authorization: Bearer` token @@ -463,14 +463,14 @@ ORDER BY embedding_count DESC; **Critical Data:** - User accounts (`users`) -- API keys (`vdb_key` in users, `api_key_encrypted` in instances) +- API keys (`embapi_key` in users, `api_key_encrypted` in instances) - ENCRYPTION_KEY environment variable (backup separately!) - Projects and embeddings **Backup Strategy:** ```bash # Full database backup -pg_dump -U postgres dhamps_vdb > backup.sql +pg_dump -U postgres embapi > backup.sql # Backup encryption key separately echo "$ENCRYPTION_KEY" > encryption_key.backup diff --git a/docs/content/reference/roadmap.md b/docs/content/reference/roadmap.md index 469aafb..299a9b6 100644 --- a/docs/content/reference/roadmap.md +++ b/docs/content/reference/roadmap.md @@ -5,11 +5,11 @@ weight: 3 # Product Roadmap -Development roadmap for dhamps-vdb, tracking completed features, in-progress work, and planned enhancements. +Development roadmap for embapi, tracking completed features, in-progress work, and planned enhancements. ## Overview -This roadmap outlines the development priorities for dhamps-vdb. Items marked with [x] are completed, items in progress are noted, and planned features are listed by priority. +This roadmap outlines the development priorities for embapi. Items marked with [x] are completed, items in progress are noted, and planned features are listed by priority. ## Completed Features @@ -329,9 +329,9 @@ Contributions are welcome! See the main repository for: We value your feedback! Please share: -- **Feature requests** - What would make dhamps-vdb more useful? +- **Feature requests** - What would make embapi more useful? - **Bug reports** - Help us improve quality -- **Use cases** - How are you using dhamps-vdb? +- **Use cases** - How are you using embapi? - **Documentation** - What needs clarification? Open an issue on GitHub or contact the maintainers directly. @@ -343,4 +343,4 @@ Open an issue on GitHub or contact the maintainers directly. - [Deployment Guide](../../deployment/) - [Reference - Configuration](configuration/) - [Reference - Database Schema](database-schema/) -- [GitHub Repository](https://github.com/mpilhlt/dhamps-vdb) +- [GitHub Repository](https://github.com/mpilhlt/embapi) diff --git a/go.mod b/go.mod index 880ef77..561e8e6 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/mpilhlt/dhamps-vdb +module github.com/mpilhlt/embapi go 1.24.0 diff --git a/internal/auth/authenticate.go b/internal/auth/authenticate.go index 48358e6..e47ae37 100644 --- a/internal/auth/authenticate.go +++ b/internal/auth/authenticate.go @@ -8,8 +8,8 @@ import ( "net/http" "strings" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgxpool" @@ -24,25 +24,25 @@ const ( // Config is the security scheme configuration for the API. var Config = map[string]*huma.SecurityScheme{ "adminAuth": { - Type: "VDBKey", + Type: "EmbAPIKey", In: "header", Scheme: "bearer", Name: "Authorization", }, "ownerAuth": { - Type: "VDBKey", + Type: "EmbAPIKey", In: "header", Scheme: "bearer", Name: "Authorization", }, "readerAuth": { - Type: "VDBKey", + Type: "EmbAPIKey", In: "header", Scheme: "bearer", Name: "Authorization", }, "editorAuth": { - Type: "VDBKey", + Type: "EmbAPIKey", In: "header", Scheme: "bearer", Name: "Authorization", @@ -82,10 +82,10 @@ func AuthTermination(api huma.API) func(ctx huma.Context, next func(huma.Context } } -// VDBKey... functions return a middleware function that checks for a valid API key. +// EmbAPIKey... functions return a middleware function that checks for a valid API key. -// VDBKeyAdminAuth checks for an admin API key in the Authorization header. -func VDBKeyAdminAuth(api huma.API, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { +// EmbAPIKeyAdminAuth checks for an admin API key in the Authorization header. +func EmbAPIKeyAdminAuth(api huma.API, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { return func(ctx huma.Context, next func(huma.Context)) { // Check if adminAuth is applicable @@ -116,8 +116,8 @@ func VDBKeyAdminAuth(api huma.API, options *models.Options) func(ctx huma.Contex } } -// VDBKeyOwnerAuth checks for an owner API key in the Authorization header. -func VDBKeyOwnerAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { +// EmbAPIKeyOwnerAuth checks for an owner API key in the Authorization header. +func EmbAPIKeyOwnerAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { return func(ctx huma.Context, next func(huma.Context)) { // Check if ownerAuth is applicable @@ -164,7 +164,7 @@ func VDBKeyOwnerAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) } // fmt.Printf(" check owner hash against API token: %s/%s ...\n", storedHash, token) - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { ctx = huma.WithValue(ctx, IsOwnerKey, true) ctx = huma.WithValue(ctx, AuthUserKey, owner) fmt.Printf(" Owner authentication successful: %s\n", owner) @@ -176,8 +176,8 @@ func VDBKeyOwnerAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) } } -// VDBKeyReaderAuth checks for a reader API key in the Authorization header. -func VDBKeyReaderAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { +// EmbAPIKeyReaderAuth checks for a reader API key in the Authorization header. +func EmbAPIKeyReaderAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { return func(ctx huma.Context, next func(huma.Context)) { // Check if readerAuth is applicable isAuthorizationRequired := false @@ -275,9 +275,9 @@ func handleProjectReaderAuth(api huma.API, pool *pgxpool.Pool, owner string, pro return } for _, authKey := range allowedKeys { - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Print(" Reader authentication successful\n") ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -314,9 +314,9 @@ func handleDefinitionReaderAuth(api huma.API, pool *pgxpool.Pool, owner string, return } for _, authKey := range allowedKeys { - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Print(" Reader authentication successful\n") ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -353,9 +353,9 @@ func handleInstanceReaderAuth(api huma.API, pool *pgxpool.Pool, owner string, in return } for _, authKey := range allowedKeys { - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Print(" Reader authentication successful\n") ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -368,9 +368,9 @@ func handleInstanceReaderAuth(api huma.API, pool *pgxpool.Pool, owner string, in } } -// VDBKeyEditorAuth checks for an editor API key in the Authorization header. +// EmbAPIKeyEditorAuth checks for an editor API key in the Authorization header. // This allows users with "editor" or "owner" role to authenticate. -func VDBKeyEditorAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { +func EmbAPIKeyEditorAuth(api huma.API, pool *pgxpool.Pool, options *models.Options) func(ctx huma.Context, next func(huma.Context)) { return func(ctx huma.Context, next func(huma.Context)) { // Check if editorAuth is applicable isAuthorizationRequired := false @@ -452,9 +452,9 @@ func handleProjectEditorAuth(api huma.API, pool *pgxpool.Pool, owner string, pro if authKey.Role != "editor" && authKey.Role != "owner" { continue } - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Printf(" Editor authentication successful (role: %s)\n", authKey.Role) ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -490,9 +490,9 @@ func handleDefinitionEditorAuth(api huma.API, pool *pgxpool.Pool, owner string, return } for _, authKey := range allowedKeys { - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Print(" Editor authentication successful\n") ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -530,9 +530,9 @@ func handleInstanceEditorAuth(api huma.API, pool *pgxpool.Pool, owner string, in if authKey.Role != "editor" && authKey.Role != "owner" { continue } - storedHash := authKey.VDBKey + storedHash := authKey.EmbAPIKey - if VDBKeyIsValid(token, storedHash) { + if EmbAPIKeyIsValid(token, storedHash) { fmt.Printf(" Editor authentication successful (role: %s)\n", authKey.Role) ctx = huma.WithValue(ctx, AuthUserKey, authKey.UserHandle) next(ctx) @@ -544,8 +544,8 @@ func handleInstanceEditorAuth(api huma.API, pool *pgxpool.Pool, owner string, in } } -// VDBKeyIsValid checks if the given API key is valid -func VDBKeyIsValid(rawKey string, storedHash string) bool { +// EmbAPIKeyIsValid checks if the given API key is valid +func EmbAPIKeyIsValid(rawKey string, storedHash string) bool { hash := sha256.Sum256([]byte(rawKey)) hashedKey := hex.EncodeToString(hash[:]) diff --git a/internal/auth/authenticate_test.go b/internal/auth/authenticate_test.go index 9389385..5bca09c 100644 --- a/internal/auth/authenticate_test.go +++ b/internal/auth/authenticate_test.go @@ -51,7 +51,7 @@ func TestApiKeyIsValid(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := VDBKeyIsValid(tt.rawKey, tt.storedHash); got != tt.want { + if got := EmbAPIKeyIsValid(tt.rawKey, tt.storedHash); got != tt.want { t.Errorf("apiKeyIsValid() = %v, want %v", got, tt.want) } }) diff --git a/internal/database/database.go b/internal/database/database.go index 25d4b67..c3d6dc7 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/mpilhlt/dhamps-vdb/internal/crypto" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/crypto" + "github.com/mpilhlt/embapi/internal/models" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -198,7 +198,7 @@ func InitializeSystemUserKey(ctx context.Context, conn *pgx.Conn) error { // Check if _system user has the placeholder key var currentKey string err = conn.QueryRow(ctx, - "SELECT vdb_key FROM users WHERE user_handle = '_system'").Scan(¤tKey) + "SELECT embapi_key FROM users WHERE user_handle = '_system'").Scan(¤tKey) if err != nil { // User doesn't exist or other error, skip silently return nil @@ -223,7 +223,7 @@ func InitializeSystemUserKey(ctx context.Context, conn *pgx.Conn) error { } _, err = conn.Exec(ctx, ` - UPDATE users SET vdb_key = $1, updated_at = NOW() WHERE user_handle = '_system' + UPDATE users SET embapi_key = $1, updated_at = NOW() WHERE user_handle = '_system' `, apiKey) if err != nil { // Don't expose the key in error messages diff --git a/internal/database/migrations/004_refactor_llm_services_architecture.sql b/internal/database/migrations/004_refactor_llm_services_architecture.sql index 47df529..ec6af5a 100644 --- a/internal/database/migrations/004_refactor_llm_services_architecture.sql +++ b/internal/database/migrations/004_refactor_llm_services_architecture.sql @@ -41,7 +41,7 @@ ON CONFLICT ("api_standard_handle") DO NOTHING; -- Step 2: Create the _system user for global definitions INSERT INTO users ("user_handle", "name", "email", "vdb_key", "created_at", "updated_at") -VALUES ('_system', 'System User', 'system@dhamps-vdb.internal', +VALUES ('_system', 'System User', 'system@embapi.internal', -- WARNING! There is a function that changes this after migrations. Make sure it does not stay like this!! '0000000000000000000000000000000000000000000000000000000000000000', NOW(), NOW()) diff --git a/internal/database/models.go b/internal/database/models.go index 3fc7e26..c6d4356 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -98,7 +98,7 @@ type User struct { UserHandle string `db:"user_handle" json:"user_handle"` Name pgtype.Text `db:"name" json:"name"` Email string `db:"email" json:"email"` - VDBKey string `db:"vdb_key" json:"vdb_key"` + EmbAPIKey string `db:"vdb_key" json:"vdb_key"` CreatedAt pgtype.Timestamp `db:"created_at" json:"created_at"` UpdatedAt pgtype.Timestamp `db:"updated_at" json:"updated_at"` } diff --git a/internal/database/queries.sql.go b/internal/database/queries.sql.go index 2c1e3d7..2d24022 100644 --- a/internal/database/queries.sql.go +++ b/internal/database/queries.sql.go @@ -719,7 +719,7 @@ type GetKeysByDefinitionParams struct { type GetKeysByDefinitionRow struct { UserHandle string `db:"user_handle" json:"user_handle"` - VDBKey string `db:"vdb_key" json:"vdb_key"` + EmbAPIKey string `db:"vdb_key" json:"vdb_key"` } func (q *Queries) GetKeysByDefinition(ctx context.Context, arg GetKeysByDefinitionParams) ([]GetKeysByDefinitionRow, error) { @@ -736,7 +736,7 @@ func (q *Queries) GetKeysByDefinition(ctx context.Context, arg GetKeysByDefiniti var items []GetKeysByDefinitionRow for rows.Next() { var i GetKeysByDefinitionRow - if err := rows.Scan(&i.UserHandle, &i.VDBKey); err != nil { + if err := rows.Scan(&i.UserHandle, &i.EmbAPIKey); err != nil { return nil, err } items = append(items, i) @@ -769,7 +769,7 @@ type GetKeysByInstanceParams struct { type GetKeysByInstanceRow struct { UserHandle string `db:"user_handle" json:"user_handle"` Role string `db:"role" json:"role"` - VDBKey string `db:"vdb_key" json:"vdb_key"` + EmbAPIKey string `db:"vdb_key" json:"vdb_key"` } func (q *Queries) GetKeysByInstance(ctx context.Context, arg GetKeysByInstanceParams) ([]GetKeysByInstanceRow, error) { @@ -786,7 +786,7 @@ func (q *Queries) GetKeysByInstance(ctx context.Context, arg GetKeysByInstancePa var items []GetKeysByInstanceRow for rows.Next() { var i GetKeysByInstanceRow - if err := rows.Scan(&i.UserHandle, &i.Role, &i.VDBKey); err != nil { + if err := rows.Scan(&i.UserHandle, &i.Role, &i.EmbAPIKey); err != nil { return nil, err } items = append(items, i) @@ -819,7 +819,7 @@ type GetKeysByProjectParams struct { type GetKeysByProjectRow struct { UserHandle string `db:"user_handle" json:"user_handle"` Role string `db:"role" json:"role"` - VDBKey string `db:"vdb_key" json:"vdb_key"` + EmbAPIKey string `db:"vdb_key" json:"vdb_key"` } func (q *Queries) GetKeysByProject(ctx context.Context, arg GetKeysByProjectParams) ([]GetKeysByProjectRow, error) { @@ -836,7 +836,7 @@ func (q *Queries) GetKeysByProject(ctx context.Context, arg GetKeysByProjectPara var items []GetKeysByProjectRow for rows.Next() { var i GetKeysByProjectRow - if err := rows.Scan(&i.UserHandle, &i.Role, &i.VDBKey); err != nil { + if err := rows.Scan(&i.UserHandle, &i.Role, &i.EmbAPIKey); err != nil { return nil, err } items = append(items, i) @@ -2165,7 +2165,7 @@ func (q *Queries) RetrieveUser(ctx context.Context, userHandle string) (User, er &i.UserHandle, &i.Name, &i.Email, - &i.VDBKey, + &i.EmbAPIKey, &i.CreatedAt, &i.UpdatedAt, ) @@ -2614,7 +2614,7 @@ type UpsertUserParams struct { UserHandle string `db:"user_handle" json:"user_handle"` Name pgtype.Text `db:"name" json:"name"` Email string `db:"email" json:"email"` - VDBKey string `db:"vdb_key" json:"vdb_key"` + EmbAPIKey string `db:"vdb_key" json:"vdb_key"` } // Generate go code with: sqlc generate @@ -2637,7 +2637,7 @@ func (q *Queries) UpsertUser(ctx context.Context, arg UpsertUserParams) (string, arg.UserHandle, arg.Name, arg.Email, - arg.VDBKey, + arg.EmbAPIKey, ) var user_handle string err := row.Scan(&user_handle) diff --git a/internal/handlers/admin.go b/internal/handlers/admin.go index 9a2f622..7b23672 100644 --- a/internal/handlers/admin.go +++ b/internal/handlers/admin.go @@ -5,8 +5,8 @@ import ( "fmt" "net/http" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgxpool" diff --git a/internal/handlers/admin_test.go b/internal/handlers/admin_test.go index 3f87700..9b671db 100644 --- a/internal/handlers/admin_test.go +++ b/internal/handlers/admin_test.go @@ -9,7 +9,7 @@ import ( "os" "testing" - "github.com/mpilhlt/dhamps-vdb/internal/handlers" + "github.com/mpilhlt/embapi/internal/handlers" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,7 @@ func TestAdminFunc(t *testing.T) { method string requestPath string bodyPath string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -39,7 +39,7 @@ func TestAdminFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/admin/footgun", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unauthorized\",\n \"status\": 401,\n \"detail\": \"Authentication failed. Perhaps a missing or incorrect API key?\"\n}\n", expectStatus: http.StatusUnauthorized, }, @@ -48,7 +48,7 @@ func TestAdminFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/admin/footgun", bodyPath: "", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "", expectStatus: http.StatusNoContent, }, @@ -78,7 +78,7 @@ func TestAdminFunc(t *testing.T) { requestURL := fmt.Sprintf("http://%v:%d%v", options.Host, options.Port, v.requestPath) req, err := http.NewRequest(v.method, requestURL, reqBody) assert.NoError(t, err) - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) resp, err := http.DefaultClient.Do(req) if err != nil { t.Errorf("Error sending request: %v\n", err) diff --git a/internal/handlers/api_standards.go b/internal/handlers/api_standards.go index 4caf28e..e2ae117 100644 --- a/internal/handlers/api_standards.go +++ b/internal/handlers/api_standards.go @@ -5,8 +5,8 @@ import ( "fmt" "net/http" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgtype" diff --git a/internal/handlers/api_standards_test.go b/internal/handlers/api_standards_test.go index e40ab22..016cbb2 100644 --- a/internal/handlers/api_standards_test.go +++ b/internal/handlers/api_standards_test.go @@ -9,7 +9,7 @@ import ( "os" "testing" - "github.com/mpilhlt/dhamps-vdb/internal/handlers" + "github.com/mpilhlt/embapi/internal/handlers" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,7 @@ func TestAPIStandardFunc(t *testing.T) { method string requestPath string bodyPath string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -39,7 +39,7 @@ func TestAPIStandardFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/api-standards/error1", bodyPath: "../../testdata/invalid_api_standard.json", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unprocessable Entity\",\n \"status\": 422,\n \"detail\": \"validation failed\",\n \"errors\": [\n {\n \"message\": \"expected required property key_field to be present\",\n \"location\": \"body\",\n \"value\": {\n \"api_standard_handle\": \"error1\",\n \"description\": \"Erroneous definition of an APi standard\",\n \"keX_method\": \"auth_bearer\"\n }\n },\n {\n \"message\": \"expected required property key_method to be present\",\n \"location\": \"body\",\n \"value\": {\n \"api_standard_handle\": \"error1\",\n \"description\": \"Erroneous definition of an APi standard\",\n \"keX_method\": \"auth_bearer\"\n }\n },\n {\n \"message\": \"unexpected property\",\n \"location\": \"body.keX_method\",\n \"value\": {\n \"api_standard_handle\": \"error1\",\n \"description\": \"Erroneous definition of an APi standard\",\n \"keX_method\": \"auth_bearer\"\n }\n }\n ]\n}\n", expectStatus: http.StatusUnprocessableEntity, }, @@ -48,7 +48,7 @@ func TestAPIStandardFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/api-standards/wrongpath", bodyPath: "../../testdata/valid_api_standard_openai_v1.json", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Bad Request\",\n \"status\": 400,\n \"detail\": \"API standard handle in URL (wrongpath) does not match handle in body (openai).\"\n}\n", expectStatus: http.StatusBadRequest, }, @@ -57,7 +57,7 @@ func TestAPIStandardFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/api-standards/test", bodyPath: "../../testdata/valid_api_standard_test.json", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UploadAPIStandardResponseBody.json\",\n \"api_standard_handle\": \"test\"\n}\n", expectStatus: http.StatusCreated, }, @@ -65,7 +65,7 @@ func TestAPIStandardFunc(t *testing.T) { name: "get all API standards", method: http.MethodGet, requestPath: "/v1/api-standards", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetAPIStandardsResponseBody.json\",\n \"api_standards\": [\n {\n \"api_standard_handle\": \"cohere\",\n \"description\": \"Cohere Embed API, Version 2, as documented in https://docs.cohere.com/reference/embed\",\n \"key_method\": \"auth_bearer\",\n \"key_field\": \"Authorization\"\n },\n {\n \"api_standard_handle\": \"gemini\",\n \"description\": \"Gemini Embeddings API, as documented in https://ai.google.dev/gemini-api/docs/embeddings\",\n \"key_method\": \"auth_bearer\",\n \"key_field\": \"x-goog-api-key\"\n },\n {\n \"api_standard_handle\": \"openai\",\n \"description\": \"OpenAI Embeddings API, Version 1, as documented in https://platform.openai.com/docs/api-reference/embeddings\",\n \"key_method\": \"auth_bearer\",\n \"key_field\": \"Authorization\"\n },\n {\n \"api_standard_handle\": \"test\",\n \"description\": \"OpenAI Embeddings API, Version 1, as documented in https://platform.openai.com/docs/api-reference/embeddings\",\n \"key_method\": \"auth_bearer\",\n \"key_field\": \"Authorization\"\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -73,7 +73,7 @@ func TestAPIStandardFunc(t *testing.T) { name: "get single API standard", method: http.MethodGet, requestPath: "/v1/api-standards/test", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"api_standard_handle\": \"test\",\n \"description\": \"OpenAI Embeddings API, Version 1, as documented in https://platform.openai.com/docs/api-reference/embeddings\",\n \"key_method\": \"auth_bearer\",\n \"key_field\": \"Authorization\"\n}\n", expectStatus: http.StatusOK, }, @@ -81,7 +81,7 @@ func TestAPIStandardFunc(t *testing.T) { name: "Delete nonexistent path", method: http.MethodDelete, requestPath: "/v1/api-standards/wrongpath", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"API standard wrongpath not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -89,7 +89,7 @@ func TestAPIStandardFunc(t *testing.T) { name: "delete API standard", method: http.MethodDelete, requestPath: "/v1/api-standards/test", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectStatus: http.StatusNoContent, }, { @@ -97,7 +97,7 @@ func TestAPIStandardFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/api-standards/openai", bodyPath: "../../testdata/valid_api_standard_openai_v1.json", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UploadAPIStandardResponseBody.json\",\n \"api_standard_handle\": \"openai\"\n}\n", expectStatus: http.StatusCreated, }, @@ -127,8 +127,8 @@ func TestAPIStandardFunc(t *testing.T) { requestURL := fmt.Sprintf("http://%v:%d%v", options.Host, options.Port, v.requestPath) req, err := http.NewRequest(v.method, requestURL, reqBody) assert.NoError(t, err) - if v.VDBKey != "" { - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + if v.EmbAPIKey != "" { + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) } resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/internal/handlers/embeddings.go b/internal/handlers/embeddings.go index 215ab1b..802d05e 100644 --- a/internal/handlers/embeddings.go +++ b/internal/handlers/embeddings.go @@ -7,8 +7,8 @@ import ( "net/http" "net/url" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgtype" diff --git a/internal/handlers/embeddings_test.go b/internal/handlers/embeddings_test.go index 7defa7c..08d4a75 100644 --- a/internal/handlers/embeddings_test.go +++ b/internal/handlers/embeddings_test.go @@ -42,7 +42,7 @@ func TestEmbeddingsFunc(t *testing.T) { } // Create LLM Service Instance to be used in embeddings tests - instanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the dhamps-vdb code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` + instanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the embapi code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` _, err = createInstance(t, instanceJSON, "alice", aliceAPIKey) if err != nil { t.Fatalf("Error creating LLM service embedding1 for testing: %v\n", err) diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index 3f73e84..6ac318e 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -14,10 +14,10 @@ import ( "testing" "time" - "github.com/mpilhlt/dhamps-vdb/internal/auth" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/handlers" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/auth" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/handlers" + "github.com/mpilhlt/embapi/internal/models" huma "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" @@ -184,10 +184,10 @@ func startTestServer(t *testing.T, pool *pgxpool.Pool, keyGen handlers.RandomKey config.Components.SecuritySchemes = auth.Config router := http.NewServeMux() api := humago.New(router, config) - api.UseMiddleware(auth.VDBKeyAdminAuth(api, &options)) - api.UseMiddleware(auth.VDBKeyOwnerAuth(api, pool, &options)) - api.UseMiddleware(auth.VDBKeyEditorAuth(api, pool, &options)) - api.UseMiddleware(auth.VDBKeyReaderAuth(api, pool, &options)) + api.UseMiddleware(auth.EmbAPIKeyAdminAuth(api, &options)) + api.UseMiddleware(auth.EmbAPIKeyOwnerAuth(api, pool, &options)) + api.UseMiddleware(auth.EmbAPIKeyEditorAuth(api, pool, &options)) + api.UseMiddleware(auth.EmbAPIKeyReaderAuth(api, pool, &options)) api.UseMiddleware(auth.AuthTermination(api)) err := handlers.AddRoutes(pool, keyGen, api) @@ -309,14 +309,14 @@ func createUser(t *testing.T, userJSON string) (string, error) { return "", err } assert.NoError(t, err) - fmt.Printf(" Successfully created user (handle: \"%s\", VDBKey: \"%s\").\n", userInfo.UserHandle, userInfo.VDBKey) + fmt.Printf(" Successfully created user (handle: \"%s\", EmbAPIKey: \"%s\").\n", userInfo.UserHandle, userInfo.EmbAPIKey) // fmt.Printf(" User info: %v\n", userInfo) - return userInfo.VDBKey, nil + return userInfo.EmbAPIKey, nil } // createProject creates a project and returns the project ID and an error value // it accepts a JSON string encoding the project object as input -func createProject(t *testing.T, projectJSON string, user string, VDBKey string) (int, error) { +func createProject(t *testing.T, projectJSON string, user string, EmbAPIKey string) (int, error) { fmt.Print(" Creating project ") jsonInput := &struct { Handle string `json:"project_handle" doc:"Handle of created or updated project"` @@ -336,7 +336,7 @@ func createProject(t *testing.T, projectJSON string, user string, VDBKey string) requestURL := fmt.Sprintf("http://%s:%d/v1/projects/%s/%s", options.Host, options.Port, user, jsonInput.Handle) requestBody := bytes.NewReader([]byte(projectJSON)) req, err := http.NewRequest(http.MethodPut, requestURL, requestBody) - req.Header.Set("Authorization", "Bearer "+VDBKey) + req.Header.Set("Authorization", "Bearer "+EmbAPIKey) assert.NoError(t, err) resp, err := http.DefaultClient.Do(req) @@ -366,7 +366,7 @@ func createProject(t *testing.T, projectJSON string, user string, VDBKey string) // createAPIStandard creates an API standard definition for testing and returns the handle and an error value // it accepts a JSON string encoding the API standard object as input -func createAPIStandard(t *testing.T, apiStandardJSON string, VDBKey string) (string, error) { +func createAPIStandard(t *testing.T, apiStandardJSON string, EmbAPIKey string) (string, error) { fmt.Print(" Creating API standard ") jsonInput := &struct { APIStandardHandle string `json:"api_standard_handle" doc:"Handle of created or updated api standard"` @@ -384,7 +384,7 @@ func createAPIStandard(t *testing.T, apiStandardJSON string, VDBKey string) (str requestURL := fmt.Sprintf("http://%s:%d/v1/api-standards/%s", options.Host, options.Port, jsonInput.APIStandardHandle) requestBody := bytes.NewReader([]byte(apiStandardJSON)) req, err := http.NewRequest(http.MethodPut, requestURL, requestBody) - req.Header.Set("Authorization", "Bearer "+VDBKey) + req.Header.Set("Authorization", "Bearer "+EmbAPIKey) assert.NoError(t, err) resp, err := http.DefaultClient.Do(req) @@ -412,7 +412,7 @@ func createAPIStandard(t *testing.T, apiStandardJSON string, VDBKey string) (str // it accepts a JSON string encoding the LLM service instance object as input // NOTE: This function is kept for backward compatibility with existing tests // It creates an LLM Service Instance (not a Definition) -func createInstance(t *testing.T, instanceJSON string, user string, VDBKey string) (string, error) { +func createInstance(t *testing.T, instanceJSON string, user string, EmbAPIKey string) (string, error) { fmt.Print(" Creating LLM service instance ") // Parse JSON to extract handle - support both old and new field names @@ -432,7 +432,7 @@ func createInstance(t *testing.T, instanceJSON string, user string, VDBKey strin requestURL := fmt.Sprintf("http://%s:%d/v1/llm-instances/%s/%s", options.Host, options.Port, user, handle) requestBody := bytes.NewReader([]byte(instanceJSON)) req, err := http.NewRequest(http.MethodPut, requestURL, requestBody) - req.Header.Set("Authorization", "Bearer "+VDBKey) + req.Header.Set("Authorization", "Bearer "+EmbAPIKey) assert.NoError(t, err) resp, err := http.DefaultClient.Do(req) @@ -466,7 +466,7 @@ func createInstance(t *testing.T, instanceJSON string, user string, VDBKey strin // createDefinition creates an LLM service definition for testing and returns the handle and an error value // it accepts a JSON string encoding the LLM service definition object as input -func createDefinition(t *testing.T, DefinitionJSON string, user string, VDBKey string) (int32, error) { +func createDefinition(t *testing.T, DefinitionJSON string, user string, EmbAPIKey string) (int32, error) { fmt.Print(" Creating LLM service definition ") jsonInput := &struct { Owner string `json:"owner" doc:"User handle of the service definition owner"` @@ -489,7 +489,7 @@ func createDefinition(t *testing.T, DefinitionJSON string, user string, VDBKey s requestURL := fmt.Sprintf("http://%s:%d/v1/llm-definitions/%s/%s", options.Host, options.Port, user, jsonInput.DefinitionHandle) requestBody := bytes.NewReader([]byte(DefinitionJSON)) req, err := http.NewRequest(http.MethodPut, requestURL, requestBody) - req.Header.Set("Authorization", "Bearer "+VDBKey) + req.Header.Set("Authorization", "Bearer "+EmbAPIKey) assert.NoError(t, err) resp, err := http.DefaultClient.Do(req) @@ -519,7 +519,7 @@ func createDefinition(t *testing.T, DefinitionJSON string, user string, VDBKey s // createEmbeddings creates some embeddings entries for testing and returns an error value // it accepts a JSON string encording the embeddings db entries -func createEmbeddings(t *testing.T, embeddings []byte, user string, Instance string, VDBKey string) error { +func createEmbeddings(t *testing.T, embeddings []byte, user string, Instance string, EmbAPIKey string) error { fmt.Print(" Creating Embeddings for testing ...\n") // Upload embeddings for similars testing @@ -528,7 +528,7 @@ func createEmbeddings(t *testing.T, embeddings []byte, user string, Instance str if err != nil { fmt.Printf("Error creating request for uploading embeddings: %v\n", err) } - req.Header.Set("Authorization", "Bearer "+VDBKey) + req.Header.Set("Authorization", "Bearer "+EmbAPIKey) req.Header.Set("Content-Type", "application/json") assert.NoError(t, err) diff --git a/internal/handlers/llm_services.go b/internal/handlers/llm_services.go index ffaae79..cc41c7d 100644 --- a/internal/handlers/llm_services.go +++ b/internal/handlers/llm_services.go @@ -7,10 +7,10 @@ import ( "os" "slices" - "github.com/mpilhlt/dhamps-vdb/internal/auth" - "github.com/mpilhlt/dhamps-vdb/internal/crypto" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/auth" + "github.com/mpilhlt/embapi/internal/crypto" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5" diff --git a/internal/handlers/llm_services_sharing_test.go b/internal/handlers/llm_services_sharing_test.go index 6f47b01..7b4a2ff 100644 --- a/internal/handlers/llm_services_sharing_test.go +++ b/internal/handlers/llm_services_sharing_test.go @@ -62,7 +62,7 @@ func TestInstanceSharingFunc(t *testing.T) { method string requestPath string bodyJSON string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -71,7 +71,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice/embedding1/share", bodyJSON: `{"share_with_handle": "charlie", "role": "reader"}`, - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Bad Request\",\n \"status\": 400,\n \"detail\": \"target user charlie does not exist: user charlie not found\"\n}\n", expectStatus: http.StatusBadRequest, }, @@ -80,7 +80,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice/nonexistent/share", bodyJSON: `{"share_with_handle": "bob", "role": "reader"}`, - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"instance alice/nonexistent not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -89,7 +89,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice/embedding1/share", bodyJSON: `{"share_with_handle": "alice", "role": "editor"}`, - VDBKey: bobAPIKey, + EmbAPIKey: bobAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unauthorized\",\n \"status\": 401,\n \"detail\": \"Authentication failed. Perhaps a missing or incorrect API key?\"\n}\n", expectStatus: http.StatusUnauthorized, }, @@ -98,7 +98,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice/embedding1/share", bodyJSON: `{"share_with_handle": "bob", "role": "reader"}`, - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ShareInstanceResponseBody.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"shared_with\": [\n {\n \"user_handle\": \"bob\",\n \"role\": \"reader\"\n }\n ]\n}\n", expectStatus: http.StatusCreated, }, @@ -107,7 +107,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice/embedding1/shared-with", bodyJSON: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetInstanceSharedUsersResponseBody.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"shared_with\": [\n {\n \"user_handle\": \"bob\",\n \"role\": \"reader\"\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -116,7 +116,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-instances/alice/embedding1/share/bob", bodyJSON: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "", expectStatus: http.StatusNoContent, }, @@ -125,7 +125,7 @@ func TestInstanceSharingFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice/embedding1/shared-with", bodyJSON: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetInstanceSharedUsersResponseBody.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"shared_with\": []\n}\n", expectStatus: http.StatusOK, }, @@ -144,7 +144,7 @@ func TestInstanceSharingFunc(t *testing.T) { assert.NoError(t, err) req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -255,7 +255,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method string requestPath string bodyJSON string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -264,7 +264,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/bob/from-definition", bodyJSON: `{"user_handle": "bob", "instance_handle": "bob-instance1", "definition_owner": "alice", "definition_handle": "openai-large", "endpoint": "https://api.openai.com/v1/embeddings", "description": "Bob's instance based on Alice's definition"}`, - VDBKey: bobAPIKey, + EmbAPIKey: bobAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Forbidden\",\n \"status\": 403,\n \"detail\": \"user does not have access to definition alice/openai-large\"\n}\n", expectStatus: http.StatusForbidden, }, @@ -273,7 +273,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/bob/from-definition", bodyJSON: `{"user_handle": "bob", "instance_handle": "bob-instance1", "definition_owner": "alice", "definition_handle": "nonexistant", "endpoint": "https://api.openai.com/v1/embeddings", "description": "Bob's instance based on Alice's definition"}`, - VDBKey: bobAPIKey, + EmbAPIKey: bobAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"definition alice/nonexistant not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -282,7 +282,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/bob/from-definition", bodyJSON: `{"user_handle": "bob", "instance_handle": "bob-instance1", "definition_owner": "_system", "definition_handle": "openai-large", "endpoint": "https://api.openai.com/v1/embeddings", "description": "Bob's instance based on _system's definition"}`, - VDBKey: bobAPIKey, + EmbAPIKey: bobAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UploadInstanceResponseBody.json\",\n \"owner\": \"bob\",\n \"instance_handle\": \"bob-instance1\",\n \"instance_id\": 1\n}\n", expectStatus: http.StatusOK, }, @@ -291,7 +291,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-definitions/alice/openai-large/share", bodyJSON: `{"share_with_handle": "bob"}`, - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ShareDefinitionResponseBody.json\",\n \"owner\": \"alice\",\n \"definition_handle\": \"openai-large\",\n \"shared_with\": [\n \"bob\"\n ]\n}\n", expectStatus: http.StatusCreated, }, @@ -300,7 +300,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-definitions/alice/openai-large/shared-with", bodyJSON: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "[\n \"bob\"\n]\n", expectStatus: http.StatusOK, }, @@ -309,7 +309,7 @@ func TestDefinitionSharingFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-definitions/alice/openai-large/share/bob", bodyJSON: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "", expectStatus: http.StatusNoContent, }, @@ -328,7 +328,7 @@ func TestDefinitionSharingFunc(t *testing.T) { assert.NoError(t, err) req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/internal/handlers/llm_services_test.go b/internal/handlers/llm_services_test.go index 9d0e93b..884209e 100644 --- a/internal/handlers/llm_services_test.go +++ b/internal/handlers/llm_services_test.go @@ -50,7 +50,7 @@ func TestInstancesFunc(t *testing.T) { method string requestPath string bodyPath string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -59,7 +59,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/llm-instances/alice/embedding1", bodyPath: "../../testdata/invalid_instance.json", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unprocessable Entity\",\n \"status\": 422,\n \"detail\": \"validation failed\",\n \"errors\": [\n {\n \"message\": \"unexpected property\",\n \"location\": \"body.api_keX\",\n \"value\": {\n \"api_keX\": \"0123456789\",\n \"api_standard\": \"openai\",\n \"description\": \"My OpenAI reduced text-embedding-3-large service\",\n \"dimensions\": 99,\n \"endpoint\": \"https://api.openai.com/v1/embeddings\",\n \"instance_handle\": \"embedding1\"\n }\n }\n ]\n}\n", expectStatus: http.StatusUnprocessableEntity, }, @@ -68,7 +68,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/llm-instances/alice/nonexistent", bodyPath: "../../testdata/valid_instance_embedding1.json", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Bad Request\",\n \"status\": 400,\n \"detail\": \"instance handle in URL (\\\"nonexistent\\\") does not match instance handle in body (\\\"embedding1\\\")\"\n}\n", expectStatus: http.StatusBadRequest, }, @@ -77,7 +77,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodPut, requestPath: "/v1/llm-instances/alice/embedding1", bodyPath: "../../testdata/valid_instance_embedding1.json", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UploadInstanceResponseBody.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1\n}\n", expectStatus: http.StatusCreated, }, @@ -86,7 +86,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice", bodyPath: "../../testdata/invalid_instance.json", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unprocessable Entity\",\n \"status\": 422,\n \"detail\": \"validation failed\",\n \"errors\": [\n {\n \"message\": \"unexpected property\",\n \"location\": \"body.api_keX\",\n \"value\": {\n \"api_keX\": \"0123456789\",\n \"api_standard\": \"openai\",\n \"description\": \"My OpenAI reduced text-embedding-3-large service\",\n \"dimensions\": 99,\n \"endpoint\": \"https://api.openai.com/v1/embeddings\",\n \"instance_handle\": \"embedding1\"\n }\n }\n ]\n}\n", expectStatus: http.StatusUnprocessableEntity, }, @@ -95,7 +95,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodPost, requestPath: "/v1/llm-instances/alice", bodyPath: "../../testdata/valid_instance_embedding1.json", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UploadInstanceResponseBody.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1\n}\n", expectStatus: http.StatusCreated, }, @@ -104,7 +104,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice", bodyPath: "", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetUserInstancesResponseBody.json\",\n \"instances\": [\n {\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -113,7 +113,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice", bodyPath: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetUserInstancesResponseBody.json\",\n \"instances\": [\n {\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -122,7 +122,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unauthorized\",\n \"status\": 401,\n \"detail\": \"Authentication failed. Perhaps a missing or incorrect API key?\"\n}\n", expectStatus: http.StatusUnauthorized, }, @@ -131,7 +131,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/john", bodyPath: "", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"user john not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -140,7 +140,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice/nonexistent", bodyPath: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"llm service nonexistent for user alice not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -149,7 +149,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice/nonexistant", bodyPath: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"llm service nonexistant for user alice not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -158,8 +158,8 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodGet, requestPath: "/v1/llm-instances/alice/embedding1", bodyPath: "", - VDBKey: aliceAPIKey, - expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/InstanceFull.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1,\n \"access_role\": \"owner\",\n \"endpoint\": \"https://api.foo.bar/v1/embed\",\n \"description\": \"An LLM Service just for testing if the dhamps-vdb code is working\",\n \"has_api_key\": true,\n \"api_standard\": \"openai\",\n \"model\": \"embed-test1\",\n \"dimensions\": 5\n}\n", + EmbAPIKey: aliceAPIKey, + expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/InstanceFull.json\",\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1,\n \"access_role\": \"owner\",\n \"endpoint\": \"https://api.foo.bar/v1/embed\",\n \"description\": \"An LLM Service just for testing if the embapi code is working\",\n \"has_api_key\": true,\n \"api_standard\": \"openai\",\n \"model\": \"embed-test1\",\n \"dimensions\": 5\n}\n", expectStatus: http.StatusOK, }, { @@ -167,7 +167,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-instances/alice/nonexistent", bodyPath: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"llm service nonexistent for user alice not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -176,7 +176,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-instances/john/embedding1", bodyPath: "", - VDBKey: options.AdminKey, + EmbAPIKey: options.AdminKey, expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Not Found\",\n \"status\": 404,\n \"detail\": \"user john not found\"\n}\n", expectStatus: http.StatusNotFound, }, @@ -185,7 +185,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-instances/alice/embedding1", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unauthorized\",\n \"status\": 401,\n \"detail\": \"Authentication failed. Perhaps a missing or incorrect API key?\"\n}\n", expectStatus: http.StatusUnauthorized, }, @@ -194,7 +194,7 @@ func TestInstancesFunc(t *testing.T) { method: http.MethodDelete, requestPath: "/v1/llm-instances/alice/embedding1", bodyPath: "", - VDBKey: aliceAPIKey, + EmbAPIKey: aliceAPIKey, expectBody: "", expectStatus: http.StatusNoContent, }, @@ -224,7 +224,7 @@ func TestInstancesFunc(t *testing.T) { requestURL := fmt.Sprintf("http://%v:%d%v", options.Host, options.Port, v.requestPath) req, err := http.NewRequest(v.method, requestURL, reqBody) assert.NoError(t, err) - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) resp, err := http.DefaultClient.Do(req) if err != nil { t.Errorf("Error sending request: %v\n", err) diff --git a/internal/handlers/projects.go b/internal/handlers/projects.go index eb9facf..3a65cb6 100644 --- a/internal/handlers/projects.go +++ b/internal/handlers/projects.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - "github.com/mpilhlt/dhamps-vdb/internal/auth" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/auth" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5" diff --git a/internal/handlers/projects_test.go b/internal/handlers/projects_test.go index 6c95447..434d246 100644 --- a/internal/handlers/projects_test.go +++ b/internal/handlers/projects_test.go @@ -41,7 +41,7 @@ func TestProjectsFunc(t *testing.T) { } // Create LLM Service Instance to be used in embeddings tests - instanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the dhamps-vdb code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` + instanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the embapi code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` _, err = createInstance(t, instanceJSON, "alice", aliceAPIKey) if err != nil { t.Fatalf("Error creating LLM service embedding1 for testing: %v\n", err) diff --git a/internal/handlers/public_access_test.go b/internal/handlers/public_access_test.go index 09e36ba..7202de4 100644 --- a/internal/handlers/public_access_test.go +++ b/internal/handlers/public_access_test.go @@ -87,7 +87,7 @@ func TestPublicAccess(t *testing.T) { method string requestPath string bodyPath string - VDBKey string + EmbAPIKey string expectBody string expectStatus int16 }{ @@ -96,7 +96,7 @@ func TestPublicAccess(t *testing.T) { method: http.MethodGet, requestPath: "/v1/projects/alice/public-test", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ProjectFull.json\",\n \"project_id\": 1,\n \"project_handle\": \"public-test\",\n \"owner\": \"alice\",\n \"description\": \"This is a test project\",\n \"public_read\": false,\n \"instance\": {\n \"owner\": \"alice\",\n \"instance_handle\": \"embedding1\",\n \"instance_id\": 1\n },\n \"role\": \"owner\",\n \"number_of_embeddings\": 3\n}\n", expectStatus: http.StatusOK, }, @@ -105,7 +105,7 @@ func TestPublicAccess(t *testing.T) { method: http.MethodGet, requestPath: "/v1/embeddings/alice/public-test", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/GetProjEmbeddingsResponseBody.json\",\n \"embeddings\": [\n {\n \"text_id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.1.1.1.1\",\n \"user_handle\": \"alice\",\n \"project_handle\": \"public-test\",\n \"project_id\": 1,\n \"instance_handle\": \"embedding1\",\n \"text\": \"This is a test document\",\n \"vector\": [\n -0.020843506,\n 0.01852417,\n 0.05328369,\n 0.07141113,\n 0.020004272\n ],\n \"vector_dim\": 5,\n \"metadata\": {\n \"author\": \"Immanuel Kant\"\n }\n },\n {\n \"text_id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.2\",\n \"user_handle\": \"alice\",\n \"project_handle\": \"public-test\",\n \"project_id\": 1,\n \"instance_handle\": \"embedding1\",\n \"text\": \"This is a similar test document\",\n \"vector\": [\n -0.020843506,\n 0.01852417,\n 0.05328369,\n 0.07141113,\n 0.020004272\n ],\n \"vector_dim\": 5,\n \"metadata\": {\n \"author\": \"Immanuel Kant\"\n }\n },\n {\n \"text_id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol2\",\n \"user_handle\": \"alice\",\n \"project_handle\": \"public-test\",\n \"project_id\": 1,\n \"instance_handle\": \"embedding1\",\n \"text\": \"This is a similar test document\",\n \"vector\": [\n -0.020843506,\n 0.01852417,\n 0.05328369,\n 0.07141113,\n 0.020004272\n ],\n \"vector_dim\": 5,\n \"metadata\": {\n \"author\": \"Immanuel Other\"\n }\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -114,7 +114,7 @@ func TestPublicAccess(t *testing.T) { method: http.MethodGet, requestPath: "/v1/embeddings/alice/public-test/https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.1.1.1.1", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/Embeddings.json\",\n \"text_id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.1.1.1.1\",\n \"user_handle\": \"alice\",\n \"project_handle\": \"public-test\",\n \"project_id\": 1,\n \"instance_handle\": \"embedding1\",\n \"text\": \"This is a test document\",\n \"vector\": [\n -0.020843506,\n 0.01852417,\n 0.05328369,\n 0.07141113,\n 0.020004272\n ],\n \"vector_dim\": 5,\n \"metadata\": {\n \"author\": \"Immanuel Kant\"\n }\n}\n", expectStatus: http.StatusOK, }, @@ -123,7 +123,7 @@ func TestPublicAccess(t *testing.T) { method: http.MethodGet, requestPath: "/v1/similars/alice/public-test/https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.1.1.1.1", bodyPath: "", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/SimilarResponseBody.json\",\n \"user_handle\": \"alice\",\n \"project_handle\": \"public-test\",\n \"results\": [\n {\n \"id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol1.2\",\n \"similarity\": 1\n },\n {\n \"id\": \"https%3A%2F%2Fid.salamanca.school%2Ftexts%2FW0001%3Avol2\",\n \"similarity\": 1\n }\n ]\n}\n", expectStatus: http.StatusOK, }, @@ -132,7 +132,7 @@ func TestPublicAccess(t *testing.T) { method: http.MethodPost, requestPath: "/v1/embeddings/alice/public-test", bodyPath: "../../testdata/valid_embeddings.json", - VDBKey: "", + EmbAPIKey: "", expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/ErrorModel.json\",\n \"title\": \"Unauthorized\",\n \"status\": 401,\n \"detail\": \"Authentication failed. Perhaps a missing or incorrect API key?\"\n}\n", expectStatus: http.StatusUnauthorized, }, @@ -162,7 +162,7 @@ func TestPublicAccess(t *testing.T) { requestURL := fmt.Sprintf("http://%v:%d%v", options.Host, options.Port, v.requestPath) req, err := http.NewRequest(v.method, requestURL, reqBody) assert.NoError(t, err) - req.Header.Set("Authorization", "Bearer "+v.VDBKey) + req.Header.Set("Authorization", "Bearer "+v.EmbAPIKey) resp, err := http.DefaultClient.Do(req) if err != nil { t.Errorf("Error sending request: %v\n", err) diff --git a/internal/handlers/similars.go b/internal/handlers/similars.go index 5e67bf0..3ecb783 100644 --- a/internal/handlers/similars.go +++ b/internal/handlers/similars.go @@ -6,8 +6,8 @@ import ( "net/http" "net/url" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgtype" diff --git a/internal/handlers/similars_test.go b/internal/handlers/similars_test.go index a678916..93d1d61 100644 --- a/internal/handlers/similars_test.go +++ b/internal/handlers/similars_test.go @@ -41,7 +41,7 @@ func TestSimilarsFunc(t *testing.T) { } // Create LLM Service - InstanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the dhamps-vdb code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` + InstanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the embapi code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` _, err = createInstance(t, InstanceJSON, "alice", aliceAPIKey) if err != nil { t.Fatalf("Error creating LLM service openai-large for testing: %v\n", err) @@ -244,7 +244,7 @@ func TestPostSimilar(t *testing.T) { } // Create LLM Service Instance with 5 dimensions - InstanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the dhamps-vdb code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` + InstanceJSON := `{ "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", "description": "An LLM Service just for testing if the embapi code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5}` _, err = createInstance(t, InstanceJSON, "alice", aliceAPIKey) if err != nil { t.Fatalf("Error creating LLM service embedding1 for testing: %v\n", err) diff --git a/internal/handlers/users.go b/internal/handlers/users.go index 55958a3..53f2044 100644 --- a/internal/handlers/users.go +++ b/internal/handlers/users.go @@ -7,8 +7,8 @@ import ( "fmt" "net/http" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2" "github.com/jackc/pgx/v5/pgtype" @@ -51,21 +51,21 @@ func putUserFunc(ctx context.Context, input *models.PutUserRequest) (*models.Upl // Create API key if user does not exist // storeKey := make([]byte, 64) var storeKey string - VDBKey := "" + EmbAPIKey := "" if u.UserHandle == input.UserHandle { // User exists, so don't create API key - storeKey = u.VDBKey + storeKey = u.EmbAPIKey fmt.Printf(" User %s already exists, stored key hash is %s.\n", input.UserHandle, storeKey) // fmt.Printf(" User %s already exists: %v.\n", input.UserHandle, u) - // fmt.Printf(" User %s. Stored key hash: '%s'.\n", input.UserHandle, u.VDBKey) - VDBKey = "not changed" + // fmt.Printf(" User %s. Stored key hash: '%s'.\n", input.UserHandle, u.EmbAPIKey) + EmbAPIKey = "not changed" } else { // User does not exist, so create a new API key - VDBKey, err = keyGen.RandomKey(32) + EmbAPIKey, err = keyGen.RandomKey(32) if err != nil { return nil, huma.Error500InternalServerError(fmt.Sprintf("unable to create API key for user %s. %v", input.UserHandle, err)) } - hash := sha256.Sum256([]byte(VDBKey)) + hash := sha256.Sum256([]byte(EmbAPIKey)) storeKey = hex.EncodeToString(hash[:]) // fmt.Printf(" Created user %s: API key %s (store hash: %s)\n", input.UserHandle, APIKey, storeKey) } @@ -73,7 +73,7 @@ func putUserFunc(ctx context.Context, input *models.PutUserRequest) (*models.Upl UserHandle: input.UserHandle, Name: pgtype.Text{String: input.Body.Name, Valid: true}, Email: input.Body.Email, - VDBKey: storeKey, + EmbAPIKey: storeKey, } // Run the query @@ -90,11 +90,11 @@ func putUserFunc(ctx context.Context, input *models.PutUserRequest) (*models.Upl response := &models.UploadUserResponse{} response.Body.UserHandle = u.UserHandle // Return the actual API key only if it was just created - // When updating an existing user, don't include the VDB key in the response - if VDBKey != "not changed" { - response.Body.VDBKey = VDBKey + // When updating an existing user, don't include the EmbAPI key in the response + if EmbAPIKey != "not changed" { + response.Body.EmbAPIKey = EmbAPIKey } else { - response.Body.VDBKey = "not changed" + response.Body.EmbAPIKey = "not changed" } return response, nil @@ -229,7 +229,7 @@ func getUserFunc(ctx context.Context, input *models.GetUserRequest) (*models.Get UserHandle: u.UserHandle, Name: u.Name.String, Email: u.Email, - VDBKey: u.VDBKey, + EmbAPIKey: u.EmbAPIKey, Projects: projects, Instances: imemberships, } diff --git a/internal/handlers/users_test.go b/internal/handlers/users_test.go index 4caa3f7..235edb6 100644 --- a/internal/handlers/users_test.go +++ b/internal/handlers/users_test.go @@ -44,7 +44,7 @@ func TestUserFunc(t *testing.T) { requestPath: "/v1/users/alice", bodyPath: "../../testdata/valid_user.json", apiKey: options.AdminKey, - expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UserResponse.json\",\n \"user_handle\": \"alice\",\n \"vdb_key\": \"12345678901234567890123456789012\"\n}\n", + expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UserResponse.json\",\n \"user_handle\": \"alice\",\n \"embapi_key\": \"12345678901234567890123456789012\"\n}\n", expectStatus: http.StatusCreated, }, { @@ -53,7 +53,7 @@ func TestUserFunc(t *testing.T) { requestPath: "/v1/users/alice", bodyPath: "", apiKey: options.AdminKey, - expectBody: "{\n \"user_handle\": \"alice\",\n \"name\": \"Alice Doe\",\n \"email\": \"alice@foo.bar\",\n \"vdb_key\": \"e1b85b27d6bcb05846c18e6a48f118e89f0c0587140de9fb3359f8370d0dba08\"\n}\n", + expectBody: "{\n \"user_handle\": \"alice\",\n \"name\": \"Alice Doe\",\n \"email\": \"alice@foo.bar\",\n \"embapi_key\": \"e1b85b27d6bcb05846c18e6a48f118e89f0c0587140de9fb3359f8370d0dba08\"\n}\n", expectStatus: http.StatusOK, }, { @@ -98,7 +98,7 @@ func TestUserFunc(t *testing.T) { requestPath: "/v1/users", bodyPath: "../../testdata/valid_user.json", apiKey: options.AdminKey, - expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UserResponse.json\",\n \"user_handle\": \"alice\",\n \"vdb_key\": \"not changed\"\n}\n", + expectBody: "{\n \"$schema\": \"http://localhost:8080/schemas/UserResponse.json\",\n \"user_handle\": \"alice\",\n \"embapi_key\": \"not changed\"\n}\n", expectStatus: http.StatusCreated, }, { diff --git a/internal/handlers/validation.go b/internal/handlers/validation.go index 643853f..8929fd6 100644 --- a/internal/handlers/validation.go +++ b/internal/handlers/validation.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/danielgtaylor/huma/v2" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/models" "github.com/xeipuuv/gojsonschema" ) diff --git a/internal/handlers/validation_unit_test.go b/internal/handlers/validation_unit_test.go index 7d93bee..a037728 100644 --- a/internal/handlers/validation_unit_test.go +++ b/internal/handlers/validation_unit_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/models" ) func TestValidateEmbeddingDimensions(t *testing.T) { diff --git a/internal/models/llm_services.go b/internal/models/llm_services.go index 3836c7b..12c62bb 100644 --- a/internal/models/llm_services.go +++ b/internal/models/llm_services.go @@ -11,15 +11,15 @@ import ( the "_system" account for all users to use, the Instances provide fully specified connectionInstance details, including personal or project API keys for Embedding service providers and can - as soon as the - respective function is implemented - be used to have the VDB forward texts to + respective function is implemented - be used to have EmbAPI forward texts to the embedding platform. This can be useful either to create the embeddings to - store in the VDB in the first place, or to encode unseen data that + store in EmbAPI in the first place, or to encode unseen data that similarities of stored embeddings can then be calculated against. Both Definitions and Instances can be shared with other users. API keys are recorded only for Instances, saved only in an encrypted way and never - displayed in any output of the VDB. (Thus, make sure to keep your own backup - copy in some secure location, don't rely on the VDB to be able to tell you + displayed in any output of EmbAPI. (Thus, make sure to keep your own backup + copy in some secure location, don't rely on EmbAPI to be able to tell you your API key in case you forget it.) With regard to terminology, the following models involve "owner" and "user_handle": diff --git a/internal/models/users.go b/internal/models/users.go index 7b76f1c..0853db8 100644 --- a/internal/models/users.go +++ b/internal/models/users.go @@ -7,7 +7,7 @@ type User struct { UserHandle string `json:"user_handle" doc:"User handle" maxLength:"20" minLength:"3" example:"jdoe"` Name string `json:"name,omitempty" doc:"User name" maxLength:"50" example:"Jane Doe"` Email string `json:"email" doc:"User email" maxLength:"100" minLength:"5" example:"foo@bar.com"` - VDBKey string `json:"vdb_key,omitempty" readOnly:"true" doc:"User API key for dhamps-vdb API" maxLength:"64" minLength:"64" example:"1234567890123456789012345678901212345678901234567890123456789012"` + EmbAPIKey string `json:"embapi_key,omitempty" readOnly:"true" doc:"User API key for EmbAPI" maxLength:"64" minLength:"64" example:"1234567890123456789012345678901212345678901234567890123456789012"` Projects ProjectMemberships `json:"projects,omitempty" readOnly:"true" doc:"Projects that the user is a member of"` Definitions Definitions `json:"definitions,omitempty" readOnly:"true" doc:"LLM Service Definitions created by the user"` Instances InstanceMemberships `json:"instances,omitempty" readOnly:"true" doc:"LLM Service Instances that the user is a member of"` @@ -61,7 +61,7 @@ type UploadUserResponse struct { type UserResponse struct { UserHandle string `json:"user_handle" doc:"Handle of created or updated user"` - VDBKey string `json:"vdb_key" doc:"VDB API key for the user"` + EmbAPIKey string `json:"embapi_key" doc:"EmbAPI key for the user"` } // Get all users diff --git a/main.go b/main.go index 0b9d665..0e925c9 100644 --- a/main.go +++ b/main.go @@ -8,10 +8,10 @@ import ( "strconv" "time" - "github.com/mpilhlt/dhamps-vdb/internal/auth" - "github.com/mpilhlt/dhamps-vdb/internal/database" - "github.com/mpilhlt/dhamps-vdb/internal/handlers" - "github.com/mpilhlt/dhamps-vdb/internal/models" + "github.com/mpilhlt/embapi/internal/auth" + "github.com/mpilhlt/embapi/internal/database" + "github.com/mpilhlt/embapi/internal/handlers" + "github.com/mpilhlt/embapi/internal/models" "github.com/danielgtaylor/huma/v2/adapters/humago" "github.com/danielgtaylor/huma/v2/autopatch" @@ -108,10 +108,10 @@ func main() { api := humago.New(router, config) api.UseMiddleware(auth.CORSMiddleware(api)) - api.UseMiddleware(auth.VDBKeyAdminAuth(api, options)) - api.UseMiddleware(auth.VDBKeyOwnerAuth(api, pool, options)) - api.UseMiddleware(auth.VDBKeyEditorAuth(api, pool, options)) - api.UseMiddleware(auth.VDBKeyReaderAuth(api, pool, options)) + api.UseMiddleware(auth.EmbAPIKeyAdminAuth(api, options)) + api.UseMiddleware(auth.EmbAPIKeyOwnerAuth(api, pool, options)) + api.UseMiddleware(auth.EmbAPIKeyEditorAuth(api, pool, options)) + api.UseMiddleware(auth.EmbAPIKeyReaderAuth(api, pool, options)) api.UseMiddleware(auth.AuthTermination(api)) // Add routes to the API diff --git a/sqlc.yaml b/sqlc.yaml index 73e510d..b202f58 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -34,9 +34,9 @@ sql: api_standard_handle: "APIStandardHandle" api_key_encrypted: "APIKeyEncrypted" has_api_key: "HasAPIKey" - vdb_key: "VDBKey" + vdb_key: "EmbAPIKey" # - column: "users.vdb_key" # go_type: # type: "[]byte" # rename: - # decode: "VDBKey" + # decode: "EmbAPIKey" diff --git a/testdata/valid_instance_embedding1.json b/testdata/valid_instance_embedding1.json index 2920660..0e59c47 100644 --- a/testdata/valid_instance_embedding1.json +++ b/testdata/valid_instance_embedding1.json @@ -1,7 +1,7 @@ { "instance_handle": "embedding1", "endpoint": "https://api.foo.bar/v1/embed", - "description": "An LLM Service just for testing if the dhamps-vdb code is working", + "description": "An LLM Service just for testing if the embapi code is working", "api_standard": "openai", "model": "embed-test1", "dimensions": 5