From 510b16fee2e01a4743da49341b7c5e6add26cfaa Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Tue, 23 Dec 2025 08:24:29 +0530 Subject: [PATCH 01/24] Introduce eventhub for api-platform --- .../pkg/eventhub/eventhub.go | 207 +++++++++++++ .../pkg/eventhub/eventhub_test.go | 281 ++++++++++++++++++ .../gateway-controller/pkg/eventhub/poller.go | 153 ++++++++++ .../gateway-controller/pkg/eventhub/store.go | 239 +++++++++++++++ .../gateway-controller/pkg/eventhub/topic.go | 111 +++++++ .../gateway-controller/pkg/eventhub/types.go | 69 +++++ .../pkg/storage/gateway-controller-db.sql | 69 ++++- 7 files changed, 1127 insertions(+), 2 deletions(-) create mode 100644 gateway/gateway-controller/pkg/eventhub/eventhub.go create mode 100644 gateway/gateway-controller/pkg/eventhub/eventhub_test.go create mode 100644 gateway/gateway-controller/pkg/eventhub/poller.go create mode 100644 gateway/gateway-controller/pkg/eventhub/store.go create mode 100644 gateway/gateway-controller/pkg/eventhub/topic.go create mode 100644 gateway/gateway-controller/pkg/eventhub/types.go diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go new file mode 100644 index 000000000..962b9d74e --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -0,0 +1,207 @@ +package eventhub + +import ( + "context" + "database/sql" + "fmt" + "sync" + "time" + + "go.uber.org/zap" +) + +// eventHub is the main implementation of EventHub interface +type eventHub struct { + db *sql.DB + store *store + registry *topicRegistry + poller *poller + config Config + logger *zap.Logger + + cleanupCtx context.Context + cleanupCancel context.CancelFunc + wg sync.WaitGroup + initialized bool + mu sync.RWMutex +} + +// New creates a new EventHub instance +func New(db *sql.DB, logger *zap.Logger, config Config) EventHub { + registry := newTopicRegistry() + store := newStore(db, logger) + + return &eventHub{ + db: db, + store: store, + registry: registry, + config: config, + logger: logger, + } +} + +// Initialize sets up the EventHub and starts background workers +func (eh *eventHub) Initialize(ctx context.Context) error { + eh.mu.Lock() + defer eh.mu.Unlock() + + if eh.initialized { + return nil + } + + eh.logger.Info("Initializing EventHub") + + // Create and start poller + eh.poller = newPoller(eh.store, eh.registry, eh.config, eh.logger) + eh.poller.start(ctx) + + // Start cleanup goroutine + eh.cleanupCtx, eh.cleanupCancel = context.WithCancel(ctx) + eh.wg.Add(1) + go eh.cleanupLoop() + + eh.initialized = true + eh.logger.Info("EventHub initialized successfully", + zap.Duration("pollInterval", eh.config.PollInterval), + ) + return nil +} + +// RegisterTopic registers a new topic with the EventHub +func (eh *eventHub) RegisterTopic(topicName TopicName) error { + ctx := context.Background() + + // Check if events table exists + exists, err := eh.store.tableExists(ctx, topicName) + if err != nil { + return fmt.Errorf("failed to check table existence: %w", err) + } + if !exists { + return fmt.Errorf("%w: table %s does not exist", + ErrTopicTableMissing, eh.store.getEventsTableName(topicName)) + } + + // Register topic in registry + if err := eh.registry.register(topicName); err != nil { + return err + } + + // Initialize empty state in database + if err := eh.store.initializeTopicState(ctx, topicName); err != nil { + return fmt.Errorf("failed to initialize state: %w", err) + } + + eh.logger.Info("Topic registered", + zap.String("topic", string(topicName)), + ) + + return nil +} + +// PublishEvent publishes an event to a topic +// Note: States and Events are updated ATOMICALLY in a transaction +func (eh *eventHub) PublishEvent(ctx context.Context, topicName TopicName, eventData []byte) error { + // Verify topic is registered + _, err := eh.registry.get(topicName) + if err != nil { + return err + } + + now := time.Now() + event := &Event{ + TopicName: topicName, + ProcessedTimestamp: now, + OriginatedTimestamp: now, + EventData: eventData, + } + + // Publish atomically (event + state update in transaction) + id, version, err := eh.store.publishEventAtomic(ctx, topicName, event) + if err != nil { + return fmt.Errorf("failed to publish event: %w", err) + } + + eh.logger.Debug("Event published", + zap.String("topic", string(topicName)), + zap.Int64("id", id), + zap.String("version", version), + ) + + return nil +} + +// RegisterSubscription registers a channel to receive events for a topic +func (eh *eventHub) RegisterSubscription(topicName TopicName, eventChan chan<- []Event) error { + if err := eh.registry.addSubscriber(topicName, eventChan); err != nil { + return err + } + + eh.logger.Info("Subscription registered", + zap.String("topic", string(topicName)), + ) + + return nil +} + +// CleanUpEvents removes events within the specified time range +func (eh *eventHub) CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error { + for _, t := range eh.registry.getAll() { + _, err := eh.store.cleanupEvents(ctx, t.name, timeFrom, timeEnd) + if err != nil { + eh.logger.Error("Failed to cleanup events for topic", + zap.String("topic", string(t.name)), + zap.Error(err), + ) + } + } + return nil +} + +// cleanupLoop runs periodic cleanup of old events +func (eh *eventHub) cleanupLoop() { + defer eh.wg.Done() + + ticker := time.NewTicker(eh.config.CleanupInterval) + defer ticker.Stop() + + for { + select { + case <-eh.cleanupCtx.Done(): + return + case <-ticker.C: + cutoff := time.Now().Add(-eh.config.RetentionPeriod) + if err := eh.store.cleanupAllTopics(eh.cleanupCtx, cutoff); err != nil { + eh.logger.Error("Periodic cleanup failed", zap.Error(err)) + } + } + } +} + +// Close gracefully shuts down the EventHub +func (eh *eventHub) Close() error { + eh.mu.Lock() + defer eh.mu.Unlock() + + if !eh.initialized { + return nil + } + + eh.logger.Info("Shutting down EventHub") + + // Stop cleanup loop + if eh.cleanupCancel != nil { + eh.cleanupCancel() + } + + // Stop poller + if eh.poller != nil { + eh.poller.stop() + } + + // Wait for goroutines + eh.wg.Wait() + + eh.initialized = false + eh.logger.Info("EventHub shutdown complete") + return nil +} diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go new file mode 100644 index 000000000..f57164358 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go @@ -0,0 +1,281 @@ +package eventhub + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func setupTestDB(t *testing.T) *sql.DB { + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + + // Create topic_states table + _, err = db.Exec(` + CREATE TABLE topic_states ( + topic_name TEXT PRIMARY KEY, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `) + require.NoError(t, err) + + // Create test events table + _, err = db.Exec(` + CREATE TABLE test_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + originated_timestamp TIMESTAMP NOT NULL, + event_data TEXT NOT NULL + ) + `) + require.NoError(t, err) + + return db +} + +func TestEventHub_RegisterTopic(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + hub := New(db, logger, DefaultConfig()) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + // Test successful registration + err = hub.RegisterTopic("test") + assert.NoError(t, err) + + // Test duplicate registration + err = hub.RegisterTopic("test") + assert.ErrorIs(t, err, ErrTopicAlreadyExists) + + // Test missing table + err = hub.RegisterTopic("nonexistent") + assert.ErrorIs(t, err, ErrTopicTableMissing) +} + +func TestEventHub_PublishAndSubscribe(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + config := DefaultConfig() + config.PollInterval = 100 * time.Millisecond // Fast polling for test + hub := New(db, logger, config) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + // Register subscription + eventChan := make(chan []Event, 10) + err = hub.RegisterSubscription("test", eventChan) + require.NoError(t, err) + + // Publish event + data, _ := json.Marshal(map[string]string{"key": "value"}) + err = hub.PublishEvent(context.Background(), "test", data) + require.NoError(t, err) + + // Wait for event delivery via polling + select { + case events := <-eventChan: + assert.GreaterOrEqual(t, len(events), 1) + assert.Equal(t, TopicName("test"), events[0].TopicName) + case <-time.After(time.Second): + t.Fatal("Timeout waiting for event") + } +} + +func TestEventHub_CleanUpEvents(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + hub := New(db, logger, DefaultConfig()) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + // Publish events + for i := 0; i < 5; i++ { + data, _ := json.Marshal(map[string]int{"index": i}) + err = hub.PublishEvent(context.Background(), "test", data) + require.NoError(t, err) + } + + // Cleanup all events + err = hub.CleanUpEvents(context.Background(), time.Time{}, time.Now().Add(time.Hour)) + require.NoError(t, err) + + // Verify events are deleted + var count int + err = db.QueryRow("SELECT COUNT(*) FROM test_events").Scan(&count) + require.NoError(t, err) + assert.Equal(t, 0, count) +} + +func TestEventHub_PollerDetectsChanges(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + config := DefaultConfig() + config.PollInterval = 50 * time.Millisecond + hub := New(db, logger, config) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + eventChan := make(chan []Event, 10) + err = hub.RegisterSubscription("test", eventChan) + require.NoError(t, err) + + // Publish multiple events + for i := 0; i < 3; i++ { + data, _ := json.Marshal(map[string]int{"index": i}) + err = hub.PublishEvent(context.Background(), "test", data) + require.NoError(t, err) + time.Sleep(10 * time.Millisecond) + } + + // Wait for events to be delivered + var receivedEvents []Event + timeout := time.After(500 * time.Millisecond) + + for { + select { + case events := <-eventChan: + receivedEvents = append(receivedEvents, events...) + if len(receivedEvents) >= 3 { + assert.Len(t, receivedEvents, 3) + return + } + case <-timeout: + t.Fatalf("Timeout: received only %d events", len(receivedEvents)) + } + } +} + +func TestEventHub_AtomicPublish(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + hub := New(db, logger, DefaultConfig()) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + // Publish event + data, _ := json.Marshal(map[string]string{"test": "data"}) + err = hub.PublishEvent(context.Background(), "test", data) + require.NoError(t, err) + + // Verify event was recorded + var eventCount int + err = db.QueryRow("SELECT COUNT(*) FROM test_events").Scan(&eventCount) + require.NoError(t, err) + assert.Equal(t, 1, eventCount) + + // Verify state was updated + var versionID string + err = db.QueryRow("SELECT version_id FROM topic_states WHERE topic_name = ?", "test").Scan(&versionID) + require.NoError(t, err) + assert.NotEmpty(t, versionID) +} + +func TestEventHub_MultipleSubscribers(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + config := DefaultConfig() + config.PollInterval = 50 * time.Millisecond + hub := New(db, logger, config) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + // Register multiple subscribers + eventChan1 := make(chan []Event, 10) + eventChan2 := make(chan []Event, 10) + err = hub.RegisterSubscription("test", eventChan1) + require.NoError(t, err) + err = hub.RegisterSubscription("test", eventChan2) + require.NoError(t, err) + + // Publish event + data, _ := json.Marshal(map[string]string{"test": "multi"}) + err = hub.PublishEvent(context.Background(), "test", data) + require.NoError(t, err) + + // Both subscribers should receive the event + timeout := time.After(time.Second) + + select { + case events := <-eventChan1: + assert.Len(t, events, 1) + case <-timeout: + t.Fatal("Timeout waiting for event on subscriber 1") + } + + select { + case events := <-eventChan2: + assert.Len(t, events, 1) + case <-timeout: + t.Fatal("Timeout waiting for event on subscriber 2") + } +} + +func TestEventHub_GracefulShutdown(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + hub := New(db, logger, DefaultConfig()) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + + err = hub.RegisterTopic("test") + require.NoError(t, err) + + // Close should complete without hanging + err = hub.Close() + assert.NoError(t, err) + + // Calling Close again should be safe + err = hub.Close() + assert.NoError(t, err) +} diff --git a/gateway/gateway-controller/pkg/eventhub/poller.go b/gateway/gateway-controller/pkg/eventhub/poller.go new file mode 100644 index 000000000..df6dd4bbc --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/poller.go @@ -0,0 +1,153 @@ +package eventhub + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" +) + +// poller handles background polling for state changes and event delivery +type poller struct { + store *store + registry *topicRegistry + config Config + logger *zap.Logger + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// newPoller creates a new event poller +func newPoller(store *store, registry *topicRegistry, config Config, logger *zap.Logger) *poller { + return &poller{ + store: store, + registry: registry, + config: config, + logger: logger, + } +} + +// start begins the poller background worker +func (p *poller) start(ctx context.Context) { + p.ctx, p.cancel = context.WithCancel(ctx) + + p.wg.Add(1) + go p.pollLoop() + + p.logger.Info("Poller started", zap.Duration("interval", p.config.PollInterval)) +} + +// pollLoop runs the main polling loop +func (p *poller) pollLoop() { + defer p.wg.Done() + + ticker := time.NewTicker(p.config.PollInterval) + defer ticker.Stop() + + for { + select { + case <-p.ctx.Done(): + return + case <-ticker.C: + p.pollAllTopics() + } + } +} + +// pollAllTopics checks all registered topics for state changes +func (p *poller) pollAllTopics() { + topics := p.registry.getAll() + + for _, t := range topics { + if err := p.pollTopic(t); err != nil { + p.logger.Error("Failed to poll topic", + zap.String("topic", string(t.name)), + zap.Error(err), + ) + } + } +} + +// pollTopic checks a single topic for state changes and delivers events +func (p *poller) pollTopic(t *topic) error { + ctx := p.ctx + + // Get current state from database + state, err := p.store.getState(ctx, t.name) + if err != nil { + return err + } + if state == nil { + // Topic state not initialized yet + return nil + } + + // Check if version has changed + if state.VersionID == t.knownVersion { + // No changes + return nil + } + + p.logger.Debug("State change detected", + zap.String("topic", string(t.name)), + zap.String("oldVersion", t.knownVersion), + zap.String("newVersion", state.VersionID), + ) + + // Fetch events since last poll + events, err := p.store.getEventsSince(ctx, t.name, t.lastPolled) + if err != nil { + return err + } + + if len(events) > 0 { + // Deliver events to subscribers + p.deliverEvents(t, events) + } + + // Update poll state + t.updatePollState(state.VersionID, time.Now()) + + return nil +} + +// deliverEvents sends events to all subscribers of a topic +func (p *poller) deliverEvents(t *topic, events []Event) { + subscribers := t.getSubscribers() + + if len(subscribers) == 0 { + p.logger.Debug("No subscribers for topic", + zap.String("topic", string(t.name)), + zap.Int("events", len(events)), + ) + return + } + + // Deliver to all subscribers + for _, ch := range subscribers { + select { + case ch <- events: + p.logger.Debug("Delivered events to subscriber", + zap.String("topic", string(t.name)), + zap.Int("events", len(events)), + ) + default: + p.logger.Warn("Subscriber channel full, dropping events", + zap.String("topic", string(t.name)), + zap.Int("events", len(events)), + ) + } + } +} + +// stop gracefully stops the poller +func (p *poller) stop() { + if p.cancel != nil { + p.cancel() + } + p.wg.Wait() + p.logger.Info("Poller stopped") +} diff --git a/gateway/gateway-controller/pkg/eventhub/store.go b/gateway/gateway-controller/pkg/eventhub/store.go new file mode 100644 index 000000000..bd1422618 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/store.go @@ -0,0 +1,239 @@ +package eventhub + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" +) + +// store handles database operations for EventHub +type store struct { + db *sql.DB + logger *zap.Logger +} + +// newStore creates a new database store +func newStore(db *sql.DB, logger *zap.Logger) *store { + return &store{ + db: db, + logger: logger, + } +} + +// tableExists checks if an events table exists for a topic +func (s *store) tableExists(ctx context.Context, topicName TopicName) (bool, error) { + tableName := s.getEventsTableName(topicName) + + query := `SELECT name FROM sqlite_master WHERE type='table' AND name=?` + var name string + err := s.db.QueryRowContext(ctx, query, tableName).Scan(&name) + + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("failed to check table existence: %w", err) + } + return true, nil +} + +// getEventsTableName returns the events table name for a topic +func (s *store) getEventsTableName(topicName TopicName) string { + return fmt.Sprintf("%s_events", string(topicName)) +} + +// initializeTopicState creates an empty state entry for a topic +func (s *store) initializeTopicState(ctx context.Context, topicName TopicName) error { + query := ` + INSERT INTO topic_states (topic_name, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(topic_name) + DO NOTHING + ` + + _, err := s.db.ExecContext(ctx, query, string(topicName), "", time.Now()) + if err != nil { + return fmt.Errorf("failed to initialize topic state: %w", err) + } + return nil +} + +// getState retrieves the current state for a topic +func (s *store) getState(ctx context.Context, topicName TopicName) (*TopicState, error) { + query := ` + SELECT topic_name, version_id, updated_at + FROM topic_states + WHERE topic_name = ? + ` + + var state TopicState + var name string + err := s.db.QueryRowContext(ctx, query, string(topicName)).Scan( + &name, &state.VersionID, &state.UpdatedAt, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get state: %w", err) + } + + state.TopicName = TopicName(name) + return &state, nil +} + +// publishEventAtomic records an event and updates state in a single transaction +func (s *store) publishEventAtomic(ctx context.Context, topicName TopicName, event *Event) (int64, string, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, "", fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + // Step 1: Insert event + tableName := s.getEventsTableName(topicName) + insertQuery := fmt.Sprintf(` + INSERT INTO %s (processed_timestamp, originated_timestamp, event_data) + VALUES (?, ?, ?) + `, tableName) + + result, err := tx.ExecContext(ctx, insertQuery, + event.ProcessedTimestamp, + event.OriginatedTimestamp, + event.EventData, + ) + if err != nil { + return 0, "", fmt.Errorf("failed to record event: %w", err) + } + + id, err := result.LastInsertId() + if err != nil { + return 0, "", fmt.Errorf("failed to get event ID: %w", err) + } + + // Step 2: Update state version + newVersion := uuid.New().String() + now := time.Now() + + updateQuery := ` + INSERT INTO topic_states (topic_name, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(topic_name) + DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at + ` + + _, err = tx.ExecContext(ctx, updateQuery, string(topicName), newVersion, now) + if err != nil { + return 0, "", fmt.Errorf("failed to update state: %w", err) + } + + // Commit transaction + if err := tx.Commit(); err != nil { + return 0, "", fmt.Errorf("failed to commit transaction: %w", err) + } + + s.logger.Debug("Published event atomically", + zap.String("topic", string(topicName)), + zap.Int64("id", id), + zap.String("version", newVersion), + ) + + return id, newVersion, nil +} + +// getEventsSince retrieves events after a given timestamp +func (s *store) getEventsSince(ctx context.Context, topicName TopicName, since time.Time) ([]Event, error) { + tableName := s.getEventsTableName(topicName) + + query := fmt.Sprintf(` + SELECT id, processed_timestamp, originated_timestamp, event_data + FROM %s + WHERE processed_timestamp > ? + ORDER BY processed_timestamp ASC + `, tableName) + + rows, err := s.db.QueryContext(ctx, query, since) + if err != nil { + return nil, fmt.Errorf("failed to query events: %w", err) + } + defer rows.Close() + + var events []Event + for rows.Next() { + var e Event + e.TopicName = topicName + if err := rows.Scan(&e.ID, &e.ProcessedTimestamp, &e.OriginatedTimestamp, &e.EventData); err != nil { + return nil, fmt.Errorf("failed to scan event: %w", err) + } + events = append(events, e) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating events: %w", err) + } + + return events, nil +} + +// cleanupEvents removes events within the specified time range +func (s *store) cleanupEvents(ctx context.Context, topicName TopicName, timeFrom, timeEnd time.Time) (int64, error) { + tableName := s.getEventsTableName(topicName) + + query := fmt.Sprintf(` + DELETE FROM %s + WHERE processed_timestamp >= ? AND processed_timestamp <= ? + `, tableName) + + result, err := s.db.ExecContext(ctx, query, timeFrom, timeEnd) + if err != nil { + return 0, fmt.Errorf("failed to cleanup events: %w", err) + } + + deleted, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get deleted count: %w", err) + } + + s.logger.Info("Cleaned up events", + zap.String("topic", string(topicName)), + zap.Int64("deleted", deleted), + zap.Time("from", timeFrom), + zap.Time("to", timeEnd), + ) + + return deleted, nil +} + +// cleanupAllTopics removes old events from all known topics +func (s *store) cleanupAllTopics(ctx context.Context, olderThan time.Time) error { + rows, err := s.db.QueryContext(ctx, `SELECT topic_name FROM topic_states`) + if err != nil { + return fmt.Errorf("failed to query topics: %w", err) + } + defer rows.Close() + + var topics []TopicName + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return fmt.Errorf("failed to scan topic name: %w", err) + } + topics = append(topics, TopicName(name)) + } + + for _, topic := range topics { + _, err := s.cleanupEvents(ctx, topic, time.Time{}, olderThan) + if err != nil { + s.logger.Warn("Failed to cleanup topic events", + zap.String("topic", string(topic)), + zap.Error(err), + ) + } + } + + return nil +} diff --git a/gateway/gateway-controller/pkg/eventhub/topic.go b/gateway/gateway-controller/pkg/eventhub/topic.go new file mode 100644 index 000000000..077504b26 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/topic.go @@ -0,0 +1,111 @@ +package eventhub + +import ( + "errors" + "sync" + "time" +) + +var ( + ErrTopicNotFound = errors.New("topic not found") + ErrTopicAlreadyExists = errors.New("topic already registered") + ErrTopicTableMissing = errors.New("events table for topic does not exist") +) + +// topic represents an internal topic with its subscriptions and poll state +type topic struct { + name TopicName + subscribers []chan<- []Event // Registered subscription channels + subscriberMu sync.RWMutex + + // Polling state + knownVersion string // Last known version from states table + lastPolled time.Time // Timestamp of last successful poll +} + +// topicRegistry manages all registered topics +type topicRegistry struct { + topics map[TopicName]*topic + mu sync.RWMutex +} + +// newTopicRegistry creates a new topic registry +func newTopicRegistry() *topicRegistry { + return &topicRegistry{ + topics: make(map[TopicName]*topic), + } +} + +// register adds a new topic to the registry +func (r *topicRegistry) register(name TopicName) error { + r.mu.Lock() + defer r.mu.Unlock() + + if _, exists := r.topics[name]; exists { + return ErrTopicAlreadyExists + } + + r.topics[name] = &topic{ + name: name, + subscribers: make([]chan<- []Event, 0), + lastPolled: time.Now(), // Start from now, don't replay old events + } + + return nil +} + +// get retrieves a topic by name +func (r *topicRegistry) get(name TopicName) (*topic, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + t, exists := r.topics[name] + if !exists { + return nil, ErrTopicNotFound + } + return t, nil +} + +// addSubscriber adds a subscription channel to a topic +func (r *topicRegistry) addSubscriber(name TopicName, ch chan<- []Event) error { + r.mu.RLock() + t, exists := r.topics[name] + r.mu.RUnlock() + + if !exists { + return ErrTopicNotFound + } + + t.subscriberMu.Lock() + defer t.subscriberMu.Unlock() + t.subscribers = append(t.subscribers, ch) + return nil +} + +// getAll returns all registered topics +func (r *topicRegistry) getAll() []*topic { + r.mu.RLock() + defer r.mu.RUnlock() + + topics := make([]*topic, 0, len(r.topics)) + for _, t := range r.topics { + topics = append(topics, t) + } + return topics +} + +// updatePollState updates the polling state for a topic +func (t *topic) updatePollState(version string, polledAt time.Time) { + t.knownVersion = version + t.lastPolled = polledAt +} + +// getSubscribers returns a copy of the subscribers list +func (t *topic) getSubscribers() []chan<- []Event { + t.subscriberMu.RLock() + defer t.subscriberMu.RUnlock() + + subs := make([]chan<- []Event, len(t.subscribers)) + copy(subs, t.subscribers) + return subs +} diff --git a/gateway/gateway-controller/pkg/eventhub/types.go b/gateway/gateway-controller/pkg/eventhub/types.go new file mode 100644 index 000000000..789eded4c --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/types.go @@ -0,0 +1,69 @@ +package eventhub + +import ( + "context" + "time" +) + +// TopicName represents a unique topic identifier +type TopicName string + +// Event represents a single event in the hub +type Event struct { + ID int64 + TopicName TopicName + ProcessedTimestamp time.Time // When event was recorded in DB + OriginatedTimestamp time.Time // When event was created + EventData []byte // JSON serialized payload +} + +// TopicState represents the version state for a topic +type TopicState struct { + TopicName TopicName + VersionID string // UUID that changes on every modification + UpdatedAt time.Time +} + +// EventHub is the main interface for the message broker +type EventHub interface { + // Initialize sets up database connections and starts background poller + Initialize(ctx context.Context) error + + // RegisterTopic registers a topic + // Returns error if the events table for this topic does not exist + // Creates entry in States table with empty version + RegisterTopic(topicName TopicName) error + + // PublishEvent publishes an event to a topic + // Updates the states table and events table atomically + PublishEvent(ctx context.Context, topicName TopicName, eventData []byte) error + + // RegisterSubscription registers a channel to receive events for a topic + // Events are delivered as batches (arrays) based on poll cycle + RegisterSubscription(topicName TopicName, eventChan chan<- []Event) error + + // CleanUpEvents removes events between the specified time range + CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error + + // Close gracefully shuts down the EventHub + Close() error +} + +// Config holds EventHub configuration +type Config struct { + // PollInterval is how often to poll for state changes + PollInterval time.Duration + // CleanupInterval is how often to run automatic cleanup + CleanupInterval time.Duration + // RetentionPeriod is how long to keep events (default 1 hour) + RetentionPeriod time.Duration +} + +// DefaultConfig returns sensible defaults +func DefaultConfig() Config { + return Config{ + PollInterval: time.Second * 5, + CleanupInterval: time.Minute * 10, + RetentionPeriod: time.Hour, + } +} diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index 9b9badac0..c97d07ca4 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -150,5 +150,70 @@ CREATE INDEX IF NOT EXISTS idx_api_key_status ON api_keys(status); CREATE INDEX IF NOT EXISTS idx_api_key_expiry ON api_keys(expires_at); CREATE INDEX IF NOT EXISTS idx_created_by ON api_keys(created_by); --- Set schema version to 5 -PRAGMA user_version = 5; +-- API Events table (added in schema version 6) +-- Stores events for API entity changes +CREATE TABLE IF NOT EXISTS api_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + originated_timestamp TIMESTAMP NOT NULL, + organization_id TEXT NOT NULL DEFAULT 'default', + action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), + entity_id TEXT NOT NULL, + event_data TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_api_events_lookup ON api_events(organization_id, processed_timestamp); + +-- Certificate Events table (added in schema version 6) +-- Stores events for certificate entity changes +CREATE TABLE IF NOT EXISTS certificate_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + originated_timestamp TIMESTAMP NOT NULL, + organization_id TEXT NOT NULL DEFAULT 'default', + action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), + entity_id TEXT NOT NULL, + event_data TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_cert_events_lookup ON certificate_events(organization_id, processed_timestamp); + +-- LLM Template Events table (added in schema version 6) +-- Stores events for LLM template entity changes +CREATE TABLE IF NOT EXISTS llm_template_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + originated_timestamp TIMESTAMP NOT NULL, + organization_id TEXT NOT NULL DEFAULT 'default', + action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), + entity_id TEXT NOT NULL, + event_data TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_llm_events_lookup ON llm_template_events(organization_id, processed_timestamp); + +-- EventHub: Topic States Table (added in schema version 7) +-- Tracks version information per topic for change detection +CREATE TABLE IF NOT EXISTS topic_states ( + topic_name TEXT PRIMARY KEY, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at); + +-- EventHub: Example events table template +-- Each topic needs its own events table following this pattern: +-- Table name format: {topic_name}_events +-- +-- Example for 'api' topic: +-- CREATE TABLE IF NOT EXISTS api_events ( +-- id INTEGER PRIMARY KEY AUTOINCREMENT, +-- processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, +-- originated_timestamp TIMESTAMP NOT NULL, +-- event_data TEXT NOT NULL +-- ); +-- CREATE INDEX IF NOT EXISTS idx_api_events_processed ON api_events(processed_timestamp); + +-- Set schema version to 7 +PRAGMA user_version = 7; From 5ecb68e4037d46a1fc7ec9024bf9d0179c75bf54 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Tue, 23 Dec 2025 11:44:59 +0530 Subject: [PATCH 02/24] Add organizationID to the topic_states table --- .../pkg/eventhub/eventhub.go | 12 +- .../pkg/eventhub/eventhub_test.go | 36 +++--- .../gateway-controller/pkg/eventhub/poller.go | 2 +- .../gateway-controller/pkg/eventhub/store.go | 36 +++--- .../gateway-controller/pkg/eventhub/topic.go | 10 +- .../gateway-controller/pkg/eventhub/types.go | 11 +- .../pkg/policyxds/snapshot.go | 2 - .../pkg/storage/gateway-controller-db.sql | 8 +- .../gateway-controller/pkg/storage/sqlite.go | 110 ++++++++++++++++++ 9 files changed, 173 insertions(+), 54 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index 962b9d74e..a2d4b7c59 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -68,7 +68,7 @@ func (eh *eventHub) Initialize(ctx context.Context) error { } // RegisterTopic registers a new topic with the EventHub -func (eh *eventHub) RegisterTopic(topicName TopicName) error { +func (eh *eventHub) RegisterTopic(organization string, topicName TopicName) error { ctx := context.Background() // Check if events table exists @@ -82,16 +82,17 @@ func (eh *eventHub) RegisterTopic(topicName TopicName) error { } // Register topic in registry - if err := eh.registry.register(topicName); err != nil { + if err := eh.registry.register(organization, topicName); err != nil { return err } // Initialize empty state in database - if err := eh.store.initializeTopicState(ctx, topicName); err != nil { + if err := eh.store.initializeTopicState(ctx, organization, topicName); err != nil { return fmt.Errorf("failed to initialize state: %w", err) } eh.logger.Info("Topic registered", + zap.String("organization", organization), zap.String("topic", string(topicName)), ) @@ -100,7 +101,7 @@ func (eh *eventHub) RegisterTopic(topicName TopicName) error { // PublishEvent publishes an event to a topic // Note: States and Events are updated ATOMICALLY in a transaction -func (eh *eventHub) PublishEvent(ctx context.Context, topicName TopicName, eventData []byte) error { +func (eh *eventHub) PublishEvent(ctx context.Context, organization string, topicName TopicName, eventData []byte) error { // Verify topic is registered _, err := eh.registry.get(topicName) if err != nil { @@ -116,12 +117,13 @@ func (eh *eventHub) PublishEvent(ctx context.Context, topicName TopicName, event } // Publish atomically (event + state update in transaction) - id, version, err := eh.store.publishEventAtomic(ctx, topicName, event) + id, version, err := eh.store.publishEventAtomic(ctx, organization, topicName, event) if err != nil { return fmt.Errorf("failed to publish event: %w", err) } eh.logger.Debug("Event published", + zap.String("organization", organization), zap.String("topic", string(topicName)), zap.Int64("id", id), zap.String("version", version), diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go index f57164358..153d7cdb5 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go @@ -20,9 +20,11 @@ func setupTestDB(t *testing.T) *sql.DB { // Create topic_states table _, err = db.Exec(` CREATE TABLE topic_states ( - topic_name TEXT PRIMARY KEY, + organization TEXT NOT NULL, + topic_name TEXT NOT NULL, version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (organization, topic_name) ) `) require.NoError(t, err) @@ -53,15 +55,15 @@ func TestEventHub_RegisterTopic(t *testing.T) { defer hub.Close() // Test successful registration - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") assert.NoError(t, err) // Test duplicate registration - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") assert.ErrorIs(t, err, ErrTopicAlreadyExists) // Test missing table - err = hub.RegisterTopic("nonexistent") + err = hub.RegisterTopic("test-org", "nonexistent") assert.ErrorIs(t, err, ErrTopicTableMissing) } @@ -78,7 +80,7 @@ func TestEventHub_PublishAndSubscribe(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) // Register subscription @@ -88,7 +90,7 @@ func TestEventHub_PublishAndSubscribe(t *testing.T) { // Publish event data, _ := json.Marshal(map[string]string{"key": "value"}) - err = hub.PublishEvent(context.Background(), "test", data) + err = hub.PublishEvent(context.Background(), "test-org", "test", data) require.NoError(t, err) // Wait for event delivery via polling @@ -112,13 +114,13 @@ func TestEventHub_CleanUpEvents(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) // Publish events for i := 0; i < 5; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test", data) + err = hub.PublishEvent(context.Background(), "test-org", "test", data) require.NoError(t, err) } @@ -146,7 +148,7 @@ func TestEventHub_PollerDetectsChanges(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) eventChan := make(chan []Event, 10) @@ -156,7 +158,7 @@ func TestEventHub_PollerDetectsChanges(t *testing.T) { // Publish multiple events for i := 0; i < 3; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test", data) + err = hub.PublishEvent(context.Background(), "test-org", "test", data) require.NoError(t, err) time.Sleep(10 * time.Millisecond) } @@ -190,12 +192,12 @@ func TestEventHub_AtomicPublish(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) // Publish event data, _ := json.Marshal(map[string]string{"test": "data"}) - err = hub.PublishEvent(context.Background(), "test", data) + err = hub.PublishEvent(context.Background(), "test-org", "test", data) require.NoError(t, err) // Verify event was recorded @@ -206,7 +208,7 @@ func TestEventHub_AtomicPublish(t *testing.T) { // Verify state was updated var versionID string - err = db.QueryRow("SELECT version_id FROM topic_states WHERE topic_name = ?", "test").Scan(&versionID) + err = db.QueryRow("SELECT version_id FROM topic_states WHERE organization = ? AND topic_name = ?", "test-org", "test").Scan(&versionID) require.NoError(t, err) assert.NotEmpty(t, versionID) } @@ -224,7 +226,7 @@ func TestEventHub_MultipleSubscribers(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) // Register multiple subscribers @@ -237,7 +239,7 @@ func TestEventHub_MultipleSubscribers(t *testing.T) { // Publish event data, _ := json.Marshal(map[string]string{"test": "multi"}) - err = hub.PublishEvent(context.Background(), "test", data) + err = hub.PublishEvent(context.Background(), "test-org", "test", data) require.NoError(t, err) // Both subscribers should receive the event @@ -268,7 +270,7 @@ func TestEventHub_GracefulShutdown(t *testing.T) { err := hub.Initialize(context.Background()) require.NoError(t, err) - err = hub.RegisterTopic("test") + err = hub.RegisterTopic("test-org", "test") require.NoError(t, err) // Close should complete without hanging diff --git a/gateway/gateway-controller/pkg/eventhub/poller.go b/gateway/gateway-controller/pkg/eventhub/poller.go index df6dd4bbc..e597d895e 100644 --- a/gateway/gateway-controller/pkg/eventhub/poller.go +++ b/gateway/gateway-controller/pkg/eventhub/poller.go @@ -76,7 +76,7 @@ func (p *poller) pollTopic(t *topic) error { ctx := p.ctx // Get current state from database - state, err := p.store.getState(ctx, t.name) + state, err := p.store.getState(ctx, t.organization, t.name) if err != nil { return err } diff --git a/gateway/gateway-controller/pkg/eventhub/store.go b/gateway/gateway-controller/pkg/eventhub/store.go index bd1422618..d7fa6f53e 100644 --- a/gateway/gateway-controller/pkg/eventhub/store.go +++ b/gateway/gateway-controller/pkg/eventhub/store.go @@ -47,15 +47,15 @@ func (s *store) getEventsTableName(topicName TopicName) string { } // initializeTopicState creates an empty state entry for a topic -func (s *store) initializeTopicState(ctx context.Context, topicName TopicName) error { +func (s *store) initializeTopicState(ctx context.Context, organization string, topicName TopicName) error { query := ` - INSERT INTO topic_states (topic_name, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(topic_name) + INSERT INTO topic_states (organization, topic_name, version_id, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(organization, topic_name) DO NOTHING ` - _, err := s.db.ExecContext(ctx, query, string(topicName), "", time.Now()) + _, err := s.db.ExecContext(ctx, query, organization, string(topicName), "", time.Now()) if err != nil { return fmt.Errorf("failed to initialize topic state: %w", err) } @@ -63,17 +63,17 @@ func (s *store) initializeTopicState(ctx context.Context, topicName TopicName) e } // getState retrieves the current state for a topic -func (s *store) getState(ctx context.Context, topicName TopicName) (*TopicState, error) { +func (s *store) getState(ctx context.Context, organization string, topicName TopicName) (*TopicState, error) { query := ` - SELECT topic_name, version_id, updated_at + SELECT organization, topic_name, version_id, updated_at FROM topic_states - WHERE topic_name = ? + WHERE organization = ? AND topic_name = ? ` var state TopicState var name string - err := s.db.QueryRowContext(ctx, query, string(topicName)).Scan( - &name, &state.VersionID, &state.UpdatedAt, + err := s.db.QueryRowContext(ctx, query, organization, string(topicName)).Scan( + &state.Organization, &name, &state.VersionID, &state.UpdatedAt, ) if err == sql.ErrNoRows { return nil, nil @@ -87,7 +87,7 @@ func (s *store) getState(ctx context.Context, topicName TopicName) (*TopicState, } // publishEventAtomic records an event and updates state in a single transaction -func (s *store) publishEventAtomic(ctx context.Context, topicName TopicName, event *Event) (int64, string, error) { +func (s *store) publishEventAtomic(ctx context.Context, organization string, topicName TopicName, event *Event) (int64, string, error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return 0, "", fmt.Errorf("failed to begin transaction: %w", err) @@ -120,13 +120,13 @@ func (s *store) publishEventAtomic(ctx context.Context, topicName TopicName, eve now := time.Now() updateQuery := ` - INSERT INTO topic_states (topic_name, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(topic_name) + INSERT INTO topic_states (organization, topic_name, version_id, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(organization, topic_name) DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at ` - _, err = tx.ExecContext(ctx, updateQuery, string(topicName), newVersion, now) + _, err = tx.ExecContext(ctx, updateQuery, organization, string(topicName), newVersion, now) if err != nil { return 0, "", fmt.Errorf("failed to update state: %w", err) } @@ -137,6 +137,7 @@ func (s *store) publishEventAtomic(ctx context.Context, topicName TopicName, eve } s.logger.Debug("Published event atomically", + zap.String("organization", organization), zap.String("topic", string(topicName)), zap.Int64("id", id), zap.String("version", newVersion), @@ -210,7 +211,7 @@ func (s *store) cleanupEvents(ctx context.Context, topicName TopicName, timeFrom // cleanupAllTopics removes old events from all known topics func (s *store) cleanupAllTopics(ctx context.Context, olderThan time.Time) error { - rows, err := s.db.QueryContext(ctx, `SELECT topic_name FROM topic_states`) + rows, err := s.db.QueryContext(ctx, `SELECT organization, topic_name FROM topic_states`) if err != nil { return fmt.Errorf("failed to query topics: %w", err) } @@ -218,8 +219,9 @@ func (s *store) cleanupAllTopics(ctx context.Context, olderThan time.Time) error var topics []TopicName for rows.Next() { + var org string var name string - if err := rows.Scan(&name); err != nil { + if err := rows.Scan(&org, &name); err != nil { return fmt.Errorf("failed to scan topic name: %w", err) } topics = append(topics, TopicName(name)) diff --git a/gateway/gateway-controller/pkg/eventhub/topic.go b/gateway/gateway-controller/pkg/eventhub/topic.go index 077504b26..4323fb56f 100644 --- a/gateway/gateway-controller/pkg/eventhub/topic.go +++ b/gateway/gateway-controller/pkg/eventhub/topic.go @@ -14,6 +14,7 @@ var ( // topic represents an internal topic with its subscriptions and poll state type topic struct { + organization string name TopicName subscribers []chan<- []Event // Registered subscription channels subscriberMu sync.RWMutex @@ -37,7 +38,7 @@ func newTopicRegistry() *topicRegistry { } // register adds a new topic to the registry -func (r *topicRegistry) register(name TopicName) error { +func (r *topicRegistry) register(organization string, name TopicName) error { r.mu.Lock() defer r.mu.Unlock() @@ -46,9 +47,10 @@ func (r *topicRegistry) register(name TopicName) error { } r.topics[name] = &topic{ - name: name, - subscribers: make([]chan<- []Event, 0), - lastPolled: time.Now(), // Start from now, don't replay old events + organization: organization, + name: name, + subscribers: make([]chan<- []Event, 0), + lastPolled: time.Now(), // Start from now, don't replay old events } return nil diff --git a/gateway/gateway-controller/pkg/eventhub/types.go b/gateway/gateway-controller/pkg/eventhub/types.go index 789eded4c..2b46bcb85 100644 --- a/gateway/gateway-controller/pkg/eventhub/types.go +++ b/gateway/gateway-controller/pkg/eventhub/types.go @@ -19,9 +19,10 @@ type Event struct { // TopicState represents the version state for a topic type TopicState struct { - TopicName TopicName - VersionID string // UUID that changes on every modification - UpdatedAt time.Time + Organization string + TopicName TopicName + VersionID string // UUID that changes on every modification + UpdatedAt time.Time } // EventHub is the main interface for the message broker @@ -32,11 +33,11 @@ type EventHub interface { // RegisterTopic registers a topic // Returns error if the events table for this topic does not exist // Creates entry in States table with empty version - RegisterTopic(topicName TopicName) error + RegisterTopic(organization string, topicName TopicName) error // PublishEvent publishes an event to a topic // Updates the states table and events table atomically - PublishEvent(ctx context.Context, topicName TopicName, eventData []byte) error + PublishEvent(ctx context.Context, organization string, topicName TopicName, eventData []byte) error // RegisterSubscription registers a channel to receive events for a topic // Events are delivered as batches (arrays) based on poll cycle diff --git a/gateway/gateway-controller/pkg/policyxds/snapshot.go b/gateway/gateway-controller/pkg/policyxds/snapshot.go index 11a0474b9..4e27d86e6 100644 --- a/gateway/gateway-controller/pkg/policyxds/snapshot.go +++ b/gateway/gateway-controller/pkg/policyxds/snapshot.go @@ -29,8 +29,6 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "go.uber.org/zap" - "google.golang.org/protobuf/types/known/anypb" - "google.golang.org/protobuf/types/known/structpb" ) // SnapshotManager manages xDS snapshots for policy configurations diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index c97d07ca4..bbf3a5976 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -195,9 +195,11 @@ CREATE INDEX IF NOT EXISTS idx_llm_events_lookup ON llm_template_events(organiza -- EventHub: Topic States Table (added in schema version 7) -- Tracks version information per topic for change detection CREATE TABLE IF NOT EXISTS topic_states ( - topic_name TEXT PRIMARY KEY, + organization TEXT NOT NULL, + topic_name TEXT NOT NULL, version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (organization, topic_name) ); CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at); @@ -216,4 +218,4 @@ CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at); -- CREATE INDEX IF NOT EXISTS idx_api_events_processed ON api_events(processed_timestamp); -- Set schema version to 7 -PRAGMA user_version = 7; +PRAGMA user_version = 8; diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index df4a7168e..670f1c2ee 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -214,6 +214,116 @@ func (s *SQLiteStorage) initSchema() error { version = 5 } + if version == 5 { + // Add event tables for EventHub + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS api_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + organization_id TEXT NOT NULL, + processed_timestamp TIMESTAMP NOT NULL, + originated_timestamp TIMESTAMP NOT NULL, + event_data TEXT NOT NULL + );`); err != nil { + return fmt.Errorf("failed to migrate schema to version 6 (api_events): %w", err) + } + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_api_events_lookup ON api_events(organization_id, processed_timestamp);`); err != nil { + return fmt.Errorf("failed to create api_events index: %w", err) + } + + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS certificate_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + organization_id TEXT NOT NULL, + processed_timestamp TIMESTAMP NOT NULL, + originated_timestamp TIMESTAMP NOT NULL, + event_data TEXT NOT NULL + );`); err != nil { + return fmt.Errorf("failed to migrate schema to version 6 (certificate_events): %w", err) + } + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_cert_events_lookup ON certificate_events(organization_id, processed_timestamp);`); err != nil { + return fmt.Errorf("failed to create certificate_events index: %w", err) + } + + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS llm_template_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + organization_id TEXT NOT NULL, + processed_timestamp TIMESTAMP NOT NULL, + originated_timestamp TIMESTAMP NOT NULL, + event_data TEXT NOT NULL + );`); err != nil { + return fmt.Errorf("failed to migrate schema to version 6 (llm_template_events): %w", err) + } + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_llm_events_lookup ON llm_template_events(organization_id, processed_timestamp);`); err != nil { + return fmt.Errorf("failed to create llm_template_events index: %w", err) + } + + if _, err := s.db.Exec("PRAGMA user_version = 6"); err != nil { + return fmt.Errorf("failed to set schema version to 6: %w", err) + } + s.logger.Info("Schema migrated to version 6 (event tables)") + version = 6 + } + + if version == 6 { + // Add topic_states table + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS topic_states ( + topic_name TEXT PRIMARY KEY, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + );`); err != nil { + return fmt.Errorf("failed to migrate schema to version 7 (topic_states): %w", err) + } + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at);`); err != nil { + return fmt.Errorf("failed to create topic_states index: %w", err) + } + if _, err := s.db.Exec("PRAGMA user_version = 7"); err != nil { + return fmt.Errorf("failed to set schema version to 7: %w", err) + } + s.logger.Info("Schema migrated to version 7 (topic_states table)") + version = 7 + } + + if version == 7 { + // Migrate topic_states to include organization as part of primary key + s.logger.Info("Migrating topic_states table to include organization (version 8)") + + // Step 1: Rename old table + if _, err := s.db.Exec(`ALTER TABLE topic_states RENAME TO topic_states_old;`); err != nil { + return fmt.Errorf("failed to rename topic_states table: %w", err) + } + + // Step 2: Create new table with organization + if _, err := s.db.Exec(`CREATE TABLE topic_states ( + organization TEXT NOT NULL, + topic_name TEXT NOT NULL, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (organization, topic_name) + );`); err != nil { + return fmt.Errorf("failed to create new topic_states table: %w", err) + } + + // Step 3: Migrate data (set organization to empty string for existing rows) + if _, err := s.db.Exec(`INSERT INTO topic_states (organization, topic_name, version_id, updated_at) + SELECT '', topic_name, version_id, updated_at FROM topic_states_old;`); err != nil { + return fmt.Errorf("failed to migrate topic_states data: %w", err) + } + + // Step 4: Drop old table + if _, err := s.db.Exec(`DROP TABLE topic_states_old;`); err != nil { + return fmt.Errorf("failed to drop old topic_states table: %w", err) + } + + // Step 5: Recreate index + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at);`); err != nil { + return fmt.Errorf("failed to create topic_states index: %w", err) + } + + if _, err := s.db.Exec("PRAGMA user_version = 8"); err != nil { + return fmt.Errorf("failed to set schema version to 8: %w", err) + } + s.logger.Info("Schema migrated to version 8 (topic_states with organization)") + version = 8 + } + s.logger.Info("Database schema up to date", zap.Int("version", version)) } From 2595d6b6fe536df93399b39410b3d137081c64b2 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Tue, 23 Dec 2025 12:59:27 +0530 Subject: [PATCH 03/24] Organization ID based events --- .../pkg/eventhub/eventhub.go | 83 +++---- .../pkg/eventhub/eventhub_test.go | 138 ++++++++--- .../gateway-controller/pkg/eventhub/poller.go | 109 +++++---- .../gateway-controller/pkg/eventhub/store.go | 219 +++++++++--------- .../gateway-controller/pkg/eventhub/topic.go | 101 ++++---- .../gateway-controller/pkg/eventhub/types.go | 54 +++-- .../gateway-controller/pkg/storage/sqlite.go | 60 +++++ 7 files changed, 438 insertions(+), 326 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index a2d4b7c59..62a625d27 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -14,7 +14,7 @@ import ( type eventHub struct { db *sql.DB store *store - registry *topicRegistry + registry *organizationRegistry poller *poller config Config logger *zap.Logger @@ -28,7 +28,7 @@ type eventHub struct { // New creates a new EventHub instance func New(db *sql.DB, logger *zap.Logger, config Config) EventHub { - registry := newTopicRegistry() + registry := newOrganizationRegistry() store := newStore(db, logger) return &eventHub{ @@ -67,94 +67,75 @@ func (eh *eventHub) Initialize(ctx context.Context) error { return nil } -// RegisterTopic registers a new topic with the EventHub -func (eh *eventHub) RegisterTopic(organization string, topicName TopicName) error { +// RegisterOrganization registers a new organization with the EventHub +func (eh *eventHub) RegisterOrganization(organizationID OrganizationID) error { ctx := context.Background() - // Check if events table exists - exists, err := eh.store.tableExists(ctx, topicName) - if err != nil { - return fmt.Errorf("failed to check table existence: %w", err) - } - if !exists { - return fmt.Errorf("%w: table %s does not exist", - ErrTopicTableMissing, eh.store.getEventsTableName(topicName)) - } - - // Register topic in registry - if err := eh.registry.register(organization, topicName); err != nil { + // Register organization in registry + if err := eh.registry.register(organizationID); err != nil { return err } // Initialize empty state in database - if err := eh.store.initializeTopicState(ctx, organization, topicName); err != nil { + if err := eh.store.initializeOrgState(ctx, organizationID); err != nil { return fmt.Errorf("failed to initialize state: %w", err) } - eh.logger.Info("Topic registered", - zap.String("organization", organization), - zap.String("topic", string(topicName)), + eh.logger.Info("Organization registered", + zap.String("organization", string(organizationID)), ) return nil } -// PublishEvent publishes an event to a topic -// Note: States and Events are updated ATOMICALLY in a transaction -func (eh *eventHub) PublishEvent(ctx context.Context, organization string, topicName TopicName, eventData []byte) error { - // Verify topic is registered - _, err := eh.registry.get(topicName) +// PublishEvent publishes an event for an organization +// Note: Organization state and events are updated ATOMICALLY in a transaction +func (eh *eventHub) PublishEvent(ctx context.Context, organizationID OrganizationID, + eventType EventType, action, entityID string, eventData []byte) error { + + // Verify organization is registered + _, err := eh.registry.get(organizationID) if err != nil { return err } - now := time.Now() - event := &Event{ - TopicName: topicName, - ProcessedTimestamp: now, - OriginatedTimestamp: now, - EventData: eventData, - } - // Publish atomically (event + state update in transaction) - id, version, err := eh.store.publishEventAtomic(ctx, organization, topicName, event) + version, err := eh.store.publishEventAtomic(ctx, organizationID, eventType, action, entityID, eventData) if err != nil { return fmt.Errorf("failed to publish event: %w", err) } eh.logger.Debug("Event published", - zap.String("organization", organization), - zap.String("topic", string(topicName)), - zap.Int64("id", id), + zap.String("organization", string(organizationID)), + zap.String("eventType", string(eventType)), + zap.String("action", action), + zap.String("entityID", entityID), zap.String("version", version), ) return nil } -// RegisterSubscription registers a channel to receive events for a topic -func (eh *eventHub) RegisterSubscription(topicName TopicName, eventChan chan<- []Event) error { - if err := eh.registry.addSubscriber(topicName, eventChan); err != nil { +// Subscribe registers a channel to receive events for an organization +// Subscriber receives ALL event types and should filter by EventType if needed +func (eh *eventHub) Subscribe(organizationID OrganizationID, eventChan chan<- []Event) error { + if err := eh.registry.addSubscriber(organizationID, eventChan); err != nil { return err } eh.logger.Info("Subscription registered", - zap.String("topic", string(topicName)), + zap.String("organization", string(organizationID)), ) return nil } -// CleanUpEvents removes events within the specified time range +// CleanUpEvents removes events from the unified events table within the specified time range func (eh *eventHub) CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error { - for _, t := range eh.registry.getAll() { - _, err := eh.store.cleanupEvents(ctx, t.name, timeFrom, timeEnd) - if err != nil { - eh.logger.Error("Failed to cleanup events for topic", - zap.String("topic", string(t.name)), - zap.Error(err), - ) - } + _, err := eh.store.cleanupEvents(ctx, timeFrom, timeEnd) + if err != nil { + eh.logger.Error("Failed to cleanup events", zap.Error(err)) + return err } return nil } @@ -172,7 +153,7 @@ func (eh *eventHub) cleanupLoop() { return case <-ticker.C: cutoff := time.Now().Add(-eh.config.RetentionPeriod) - if err := eh.store.cleanupAllTopics(eh.cleanupCtx, cutoff); err != nil { + if err := eh.store.cleanupAllOrganizations(eh.cleanupCtx, cutoff); err != nil { eh.logger.Error("Periodic cleanup failed", zap.Error(err)) } } diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go index 153d7cdb5..1010d323e 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go @@ -17,25 +17,27 @@ func setupTestDB(t *testing.T) *sql.DB { db, err := sql.Open("sqlite3", ":memory:") require.NoError(t, err) - // Create topic_states table + // Create organization_states table _, err = db.Exec(` - CREATE TABLE topic_states ( - organization TEXT NOT NULL, - topic_name TEXT NOT NULL, + CREATE TABLE organization_states ( + organization TEXT PRIMARY KEY, version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (organization, topic_name) + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ) `) require.NoError(t, err) - // Create test events table + // Create unified events table _, err = db.Exec(` - CREATE TABLE test_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + CREATE TABLE events ( + organization_id TEXT NOT NULL, processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, originated_timestamp TIMESTAMP NOT NULL, - event_data TEXT NOT NULL + event_type TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), + entity_id TEXT NOT NULL, + event_data TEXT NOT NULL, + PRIMARY KEY (organization_id, processed_timestamp) ) `) require.NoError(t, err) @@ -43,7 +45,7 @@ func setupTestDB(t *testing.T) *sql.DB { return db } -func TestEventHub_RegisterTopic(t *testing.T) { +func TestEventHub_RegisterOrganization(t *testing.T) { db := setupTestDB(t) defer db.Close() @@ -55,16 +57,12 @@ func TestEventHub_RegisterTopic(t *testing.T) { defer hub.Close() // Test successful registration - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") assert.NoError(t, err) // Test duplicate registration - err = hub.RegisterTopic("test-org", "test") - assert.ErrorIs(t, err, ErrTopicAlreadyExists) - - // Test missing table - err = hub.RegisterTopic("test-org", "nonexistent") - assert.ErrorIs(t, err, ErrTopicTableMissing) + err = hub.RegisterOrganization("test-org") + assert.ErrorIs(t, err, ErrOrganizationAlreadyExists) } func TestEventHub_PublishAndSubscribe(t *testing.T) { @@ -80,24 +78,27 @@ func TestEventHub_PublishAndSubscribe(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) // Register subscription eventChan := make(chan []Event, 10) - err = hub.RegisterSubscription("test", eventChan) + err = hub.Subscribe("test-org", eventChan) require.NoError(t, err) // Publish event data, _ := json.Marshal(map[string]string{"key": "value"}) - err = hub.PublishEvent(context.Background(), "test-org", "test", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) require.NoError(t, err) // Wait for event delivery via polling select { case events := <-eventChan: assert.GreaterOrEqual(t, len(events), 1) - assert.Equal(t, TopicName("test"), events[0].TopicName) + assert.Equal(t, OrganizationID("test-org"), events[0].OrganizationID) + assert.Equal(t, EventTypeAPI, events[0].EventType) + assert.Equal(t, "CREATE", events[0].Action) + assert.Equal(t, "api-1", events[0].EntityID) case <-time.After(time.Second): t.Fatal("Timeout waiting for event") } @@ -114,13 +115,13 @@ func TestEventHub_CleanUpEvents(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) // Publish events for i := 0; i < 5; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test-org", "test", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) require.NoError(t, err) } @@ -130,7 +131,7 @@ func TestEventHub_CleanUpEvents(t *testing.T) { // Verify events are deleted var count int - err = db.QueryRow("SELECT COUNT(*) FROM test_events").Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM events").Scan(&count) require.NoError(t, err) assert.Equal(t, 0, count) } @@ -148,17 +149,17 @@ func TestEventHub_PollerDetectsChanges(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) eventChan := make(chan []Event, 10) - err = hub.RegisterSubscription("test", eventChan) + err = hub.Subscribe("test-org", eventChan) require.NoError(t, err) // Publish multiple events for i := 0; i < 3; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test-org", "test", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) require.NoError(t, err) time.Sleep(10 * time.Millisecond) } @@ -192,23 +193,23 @@ func TestEventHub_AtomicPublish(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) // Publish event data, _ := json.Marshal(map[string]string{"test": "data"}) - err = hub.PublishEvent(context.Background(), "test-org", "test", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) require.NoError(t, err) - // Verify event was recorded + // Verify event was recorded in unified table var eventCount int - err = db.QueryRow("SELECT COUNT(*) FROM test_events").Scan(&eventCount) + err = db.QueryRow("SELECT COUNT(*) FROM events WHERE organization_id = ?", "test-org").Scan(&eventCount) require.NoError(t, err) assert.Equal(t, 1, eventCount) // Verify state was updated var versionID string - err = db.QueryRow("SELECT version_id FROM topic_states WHERE organization = ? AND topic_name = ?", "test-org", "test").Scan(&versionID) + err = db.QueryRow("SELECT version_id FROM organization_states WHERE organization = ?", "test-org").Scan(&versionID) require.NoError(t, err) assert.NotEmpty(t, versionID) } @@ -226,20 +227,20 @@ func TestEventHub_MultipleSubscribers(t *testing.T) { require.NoError(t, err) defer hub.Close() - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) // Register multiple subscribers eventChan1 := make(chan []Event, 10) eventChan2 := make(chan []Event, 10) - err = hub.RegisterSubscription("test", eventChan1) + err = hub.Subscribe("test-org", eventChan1) require.NoError(t, err) - err = hub.RegisterSubscription("test", eventChan2) + err = hub.Subscribe("test-org", eventChan2) require.NoError(t, err) // Publish event data, _ := json.Marshal(map[string]string{"test": "multi"}) - err = hub.PublishEvent(context.Background(), "test-org", "test", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) require.NoError(t, err) // Both subscribers should receive the event @@ -270,7 +271,7 @@ func TestEventHub_GracefulShutdown(t *testing.T) { err := hub.Initialize(context.Background()) require.NoError(t, err) - err = hub.RegisterTopic("test-org", "test") + err = hub.RegisterOrganization("test-org") require.NoError(t, err) // Close should complete without hanging @@ -281,3 +282,64 @@ func TestEventHub_GracefulShutdown(t *testing.T) { err = hub.Close() assert.NoError(t, err) } + +func TestEventHub_MultipleEventTypes(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + logger := zap.NewNop() + config := DefaultConfig() + config.PollInterval = 50 * time.Millisecond + hub := New(db, logger, config) + + err := hub.Initialize(context.Background()) + require.NoError(t, err) + defer hub.Close() + + err = hub.RegisterOrganization("test-org") + require.NoError(t, err) + + eventChan := make(chan []Event, 10) + err = hub.Subscribe("test-org", eventChan) + require.NoError(t, err) + + // Publish different event types + data1, _ := json.Marshal(map[string]string{"type": "api"}) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data1) + require.NoError(t, err) + + data2, _ := json.Marshal(map[string]string{"type": "cert"}) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeCertificate, "UPDATE", "cert-1", data2) + require.NoError(t, err) + + data3, _ := json.Marshal(map[string]string{"type": "llm"}) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeLLMTemplate, "DELETE", "template-1", data3) + require.NoError(t, err) + + // Wait for events to be delivered (all types should come through) + var receivedEvents []Event + timeout := time.After(time.Second) + + for { + select { + case events := <-eventChan: + receivedEvents = append(receivedEvents, events...) + if len(receivedEvents) >= 3 { + // Verify all event types were received + assert.Len(t, receivedEvents, 3) + + eventTypeMap := make(map[EventType]bool) + for _, e := range receivedEvents { + eventTypeMap[e.EventType] = true + } + + assert.True(t, eventTypeMap[EventTypeAPI]) + assert.True(t, eventTypeMap[EventTypeCertificate]) + assert.True(t, eventTypeMap[EventTypeLLMTemplate]) + return + } + case <-timeout: + t.Fatalf("Timeout: received only %d events", len(receivedEvents)) + } + } +} diff --git a/gateway/gateway-controller/pkg/eventhub/poller.go b/gateway/gateway-controller/pkg/eventhub/poller.go index e597d895e..cbf678b9b 100644 --- a/gateway/gateway-controller/pkg/eventhub/poller.go +++ b/gateway/gateway-controller/pkg/eventhub/poller.go @@ -11,7 +11,7 @@ import ( // poller handles background polling for state changes and event delivery type poller struct { store *store - registry *topicRegistry + registry *organizationRegistry config Config logger *zap.Logger @@ -21,7 +21,7 @@ type poller struct { } // newPoller creates a new event poller -func newPoller(store *store, registry *topicRegistry, config Config, logger *zap.Logger) *poller { +func newPoller(store *store, registry *organizationRegistry, config Config, logger *zap.Logger) *poller { return &poller{ store: store, registry: registry, @@ -52,91 +52,88 @@ func (p *poller) pollLoop() { case <-p.ctx.Done(): return case <-ticker.C: - p.pollAllTopics() + p.pollAllOrganizations() } } } -// pollAllTopics checks all registered topics for state changes -func (p *poller) pollAllTopics() { - topics := p.registry.getAll() - - for _, t := range topics { - if err := p.pollTopic(t); err != nil { - p.logger.Error("Failed to poll topic", - zap.String("topic", string(t.name)), - zap.Error(err), - ) - } - } -} - -// pollTopic checks a single topic for state changes and delivers events -func (p *poller) pollTopic(t *topic) error { +// pollAllOrganizations checks all organizations for state changes using single query +func (p *poller) pollAllOrganizations() { ctx := p.ctx - // Get current state from database - state, err := p.store.getState(ctx, t.organization, t.name) + // STEP 1: Single query for ALL organization states + states, err := p.store.getAllStates(ctx) if err != nil { - return err - } - if state == nil { - // Topic state not initialized yet - return nil + p.logger.Error("Failed to fetch all states", zap.Error(err)) + return } - // Check if version has changed - if state.VersionID == t.knownVersion { - // No changes - return nil - } + // STEP 2: Loop through each organization sequentially + for _, state := range states { + orgID := OrganizationID(state.Organization) - p.logger.Debug("State change detected", - zap.String("topic", string(t.name)), - zap.String("oldVersion", t.knownVersion), - zap.String("newVersion", state.VersionID), - ) + // Get the organization from registry + org, err := p.registry.get(orgID) + if err != nil { + // Organization not registered with subscribers, skip + continue + } - // Fetch events since last poll - events, err := p.store.getEventsSince(ctx, t.name, t.lastPolled) - if err != nil { - return err - } + // Check if version changed + if state.VersionID == org.knownVersion { + // No changes + continue + } - if len(events) > 0 { - // Deliver events to subscribers - p.deliverEvents(t, events) - } + p.logger.Debug("State change detected", + zap.String("organization", string(orgID)), + zap.String("oldVersion", org.knownVersion), + zap.String("newVersion", state.VersionID), + ) - // Update poll state - t.updatePollState(state.VersionID, time.Now()) + // Fetch events since last poll + events, err := p.store.getEventsSince(ctx, orgID, org.lastPolled) + if err != nil { + p.logger.Error("Failed to fetch events", + zap.String("organization", string(orgID)), + zap.Error(err)) + continue + } + + if len(events) > 0 { + // Deliver events to subscribers + p.deliverEvents(org, events) + } - return nil + // Update poll state + org.updatePollState(state.VersionID, time.Now()) + } } -// deliverEvents sends events to all subscribers of a topic -func (p *poller) deliverEvents(t *topic, events []Event) { - subscribers := t.getSubscribers() +// deliverEvents sends events to all subscribers of an organization +func (p *poller) deliverEvents(org *organization, events []Event) { + subscribers := org.getSubscribers() if len(subscribers) == 0 { - p.logger.Debug("No subscribers for topic", - zap.String("topic", string(t.name)), + p.logger.Debug("No subscribers for organization", + zap.String("organization", string(org.id)), zap.Int("events", len(events)), ) return } - // Deliver to all subscribers + // Deliver ALL events (all event types) to subscribers + // Consumers are responsible for filtering by EventType if needed for _, ch := range subscribers { select { case ch <- events: p.logger.Debug("Delivered events to subscriber", - zap.String("topic", string(t.name)), + zap.String("organization", string(org.id)), zap.Int("events", len(events)), ) default: p.logger.Warn("Subscriber channel full, dropping events", - zap.String("topic", string(t.name)), + zap.String("organization", string(org.id)), zap.Int("events", len(events)), ) } diff --git a/gateway/gateway-controller/pkg/eventhub/store.go b/gateway/gateway-controller/pkg/eventhub/store.go index d7fa6f53e..f9357e868 100644 --- a/gateway/gateway-controller/pkg/eventhub/store.go +++ b/gateway/gateway-controller/pkg/eventhub/store.go @@ -24,56 +24,63 @@ func newStore(db *sql.DB, logger *zap.Logger) *store { } } -// tableExists checks if an events table exists for a topic -func (s *store) tableExists(ctx context.Context, topicName TopicName) (bool, error) { - tableName := s.getEventsTableName(topicName) - - query := `SELECT name FROM sqlite_master WHERE type='table' AND name=?` - var name string - err := s.db.QueryRowContext(ctx, query, tableName).Scan(&name) +// initializeOrgState creates an empty state entry for an organization +func (s *store) initializeOrgState(ctx context.Context, orgID OrganizationID) error { + query := ` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO NOTHING + ` - if err == sql.ErrNoRows { - return false, nil - } + _, err := s.db.ExecContext(ctx, query, string(orgID), "", time.Now()) if err != nil { - return false, fmt.Errorf("failed to check table existence: %w", err) + return fmt.Errorf("failed to initialize organization state: %w", err) } - return true, nil -} - -// getEventsTableName returns the events table name for a topic -func (s *store) getEventsTableName(topicName TopicName) string { - return fmt.Sprintf("%s_events", string(topicName)) + return nil } -// initializeTopicState creates an empty state entry for a topic -func (s *store) initializeTopicState(ctx context.Context, organization string, topicName TopicName) error { +// getAllStates retrieves all organization states in a single query +func (s *store) getAllStates(ctx context.Context) ([]OrganizationState, error) { query := ` - INSERT INTO topic_states (organization, topic_name, version_id, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(organization, topic_name) - DO NOTHING + SELECT organization, version_id, updated_at + FROM organization_states + ORDER BY organization ` - _, err := s.db.ExecContext(ctx, query, organization, string(topicName), "", time.Now()) + rows, err := s.db.QueryContext(ctx, query) if err != nil { - return fmt.Errorf("failed to initialize topic state: %w", err) + return nil, fmt.Errorf("failed to query all states: %w", err) } - return nil + defer rows.Close() + + var states []OrganizationState + for rows.Next() { + var state OrganizationState + if err := rows.Scan(&state.Organization, &state.VersionID, &state.UpdatedAt); err != nil { + return nil, fmt.Errorf("failed to scan state: %w", err) + } + states = append(states, state) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating states: %w", err) + } + + return states, nil } -// getState retrieves the current state for a topic -func (s *store) getState(ctx context.Context, organization string, topicName TopicName) (*TopicState, error) { +// getState retrieves the current state for an organization +func (s *store) getState(ctx context.Context, orgID OrganizationID) (*OrganizationState, error) { query := ` - SELECT organization, topic_name, version_id, updated_at - FROM topic_states - WHERE organization = ? AND topic_name = ? + SELECT organization, version_id, updated_at + FROM organization_states + WHERE organization = ? ` - var state TopicState - var name string - err := s.db.QueryRowContext(ctx, query, organization, string(topicName)).Scan( - &state.Organization, &name, &state.VersionID, &state.UpdatedAt, + var state OrganizationState + err := s.db.QueryRowContext(ctx, query, string(orgID)).Scan( + &state.Organization, &state.VersionID, &state.UpdatedAt, ) if err == sql.ErrNoRows { return nil, nil @@ -82,82 +89,83 @@ func (s *store) getState(ctx context.Context, organization string, topicName Top return nil, fmt.Errorf("failed to get state: %w", err) } - state.TopicName = TopicName(name) return &state, nil } // publishEventAtomic records an event and updates state in a single transaction -func (s *store) publishEventAtomic(ctx context.Context, organization string, topicName TopicName, event *Event) (int64, string, error) { +func (s *store) publishEventAtomic(ctx context.Context, orgID OrganizationID, eventType EventType, + action, entityID string, eventData []byte) (string, error) { + tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return 0, "", fmt.Errorf("failed to begin transaction: %w", err) + return "", fmt.Errorf("failed to begin transaction: %w", err) } defer tx.Rollback() - // Step 1: Insert event - tableName := s.getEventsTableName(topicName) - insertQuery := fmt.Sprintf(` - INSERT INTO %s (processed_timestamp, originated_timestamp, event_data) - VALUES (?, ?, ?) - `, tableName) + now := time.Now() - result, err := tx.ExecContext(ctx, insertQuery, - event.ProcessedTimestamp, - event.OriginatedTimestamp, - event.EventData, - ) - if err != nil { - return 0, "", fmt.Errorf("failed to record event: %w", err) - } + // Step 1: Insert event into unified events table + insertQuery := ` + INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, + event_type, action, entity_id, event_data) + VALUES (?, ?, ?, ?, ?, ?, ?) + ` - id, err := result.LastInsertId() + _, err = tx.ExecContext(ctx, insertQuery, + string(orgID), + now, + now, + string(eventType), + action, + entityID, + eventData, + ) if err != nil { - return 0, "", fmt.Errorf("failed to get event ID: %w", err) + return "", fmt.Errorf("failed to record event: %w", err) } - // Step 2: Update state version + // Step 2: Update organization state version newVersion := uuid.New().String() - now := time.Now() updateQuery := ` - INSERT INTO topic_states (organization, topic_name, version_id, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(organization, topic_name) + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at ` - _, err = tx.ExecContext(ctx, updateQuery, organization, string(topicName), newVersion, now) + _, err = tx.ExecContext(ctx, updateQuery, string(orgID), newVersion, now) if err != nil { - return 0, "", fmt.Errorf("failed to update state: %w", err) + return "", fmt.Errorf("failed to update state: %w", err) } // Commit transaction if err := tx.Commit(); err != nil { - return 0, "", fmt.Errorf("failed to commit transaction: %w", err) + return "", fmt.Errorf("failed to commit transaction: %w", err) } s.logger.Debug("Published event atomically", - zap.String("organization", organization), - zap.String("topic", string(topicName)), - zap.Int64("id", id), + zap.String("organization", string(orgID)), + zap.String("eventType", string(eventType)), + zap.String("action", action), + zap.String("entityID", entityID), zap.String("version", newVersion), ) - return id, newVersion, nil + return newVersion, nil } -// getEventsSince retrieves events after a given timestamp -func (s *store) getEventsSince(ctx context.Context, topicName TopicName, since time.Time) ([]Event, error) { - tableName := s.getEventsTableName(topicName) - - query := fmt.Sprintf(` - SELECT id, processed_timestamp, originated_timestamp, event_data - FROM %s - WHERE processed_timestamp > ? +// getEventsSince retrieves events for an organization after a given timestamp +func (s *store) getEventsSince(ctx context.Context, orgID OrganizationID, since time.Time) ([]Event, error) { + query := ` + SELECT processed_timestamp, originated_timestamp, event_type, + action, entity_id, event_data + FROM events + WHERE organization_id = ? AND processed_timestamp > ? ORDER BY processed_timestamp ASC - `, tableName) + ` - rows, err := s.db.QueryContext(ctx, query, since) + rows, err := s.db.QueryContext(ctx, query, string(orgID), since) if err != nil { return nil, fmt.Errorf("failed to query events: %w", err) } @@ -166,10 +174,14 @@ func (s *store) getEventsSince(ctx context.Context, topicName TopicName, since t var events []Event for rows.Next() { var e Event - e.TopicName = topicName - if err := rows.Scan(&e.ID, &e.ProcessedTimestamp, &e.OriginatedTimestamp, &e.EventData); err != nil { + var eventTypeStr string + e.OrganizationID = orgID + + if err := rows.Scan(&e.ProcessedTimestamp, &e.OriginatedTimestamp, + &eventTypeStr, &e.Action, &e.EntityID, &e.EventData); err != nil { return nil, fmt.Errorf("failed to scan event: %w", err) } + e.EventType = EventType(eventTypeStr) events = append(events, e) } @@ -180,14 +192,12 @@ func (s *store) getEventsSince(ctx context.Context, topicName TopicName, since t return events, nil } -// cleanupEvents removes events within the specified time range -func (s *store) cleanupEvents(ctx context.Context, topicName TopicName, timeFrom, timeEnd time.Time) (int64, error) { - tableName := s.getEventsTableName(topicName) - - query := fmt.Sprintf(` - DELETE FROM %s +// cleanupEvents removes events from the unified table within the specified time range +func (s *store) cleanupEvents(ctx context.Context, timeFrom, timeEnd time.Time) (int64, error) { + query := ` + DELETE FROM events WHERE processed_timestamp >= ? AND processed_timestamp <= ? - `, tableName) + ` result, err := s.db.ExecContext(ctx, query, timeFrom, timeEnd) if err != nil { @@ -200,7 +210,6 @@ func (s *store) cleanupEvents(ctx context.Context, topicName TopicName, timeFrom } s.logger.Info("Cleaned up events", - zap.String("topic", string(topicName)), zap.Int64("deleted", deleted), zap.Time("from", timeFrom), zap.Time("to", timeEnd), @@ -209,33 +218,27 @@ func (s *store) cleanupEvents(ctx context.Context, topicName TopicName, timeFrom return deleted, nil } -// cleanupAllTopics removes old events from all known topics -func (s *store) cleanupAllTopics(ctx context.Context, olderThan time.Time) error { - rows, err := s.db.QueryContext(ctx, `SELECT organization, topic_name FROM topic_states`) +// cleanupAllOrganizations removes old events from the unified events table +func (s *store) cleanupAllOrganizations(ctx context.Context, olderThan time.Time) error { + query := ` + DELETE FROM events + WHERE processed_timestamp < ? + ` + + result, err := s.db.ExecContext(ctx, query, olderThan) if err != nil { - return fmt.Errorf("failed to query topics: %w", err) + return fmt.Errorf("failed to cleanup old events: %w", err) } - defer rows.Close() - var topics []TopicName - for rows.Next() { - var org string - var name string - if err := rows.Scan(&org, &name); err != nil { - return fmt.Errorf("failed to scan topic name: %w", err) - } - topics = append(topics, TopicName(name)) + deleted, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get deleted count: %w", err) } - for _, topic := range topics { - _, err := s.cleanupEvents(ctx, topic, time.Time{}, olderThan) - if err != nil { - s.logger.Warn("Failed to cleanup topic events", - zap.String("topic", string(topic)), - zap.Error(err), - ) - } - } + s.logger.Info("Cleaned up old events across all organizations", + zap.Int64("deleted", deleted), + zap.Time("olderThan", olderThan), + ) return nil } diff --git a/gateway/gateway-controller/pkg/eventhub/topic.go b/gateway/gateway-controller/pkg/eventhub/topic.go index 4323fb56f..7af18ef0a 100644 --- a/gateway/gateway-controller/pkg/eventhub/topic.go +++ b/gateway/gateway-controller/pkg/eventhub/topic.go @@ -7,107 +7,104 @@ import ( ) var ( - ErrTopicNotFound = errors.New("topic not found") - ErrTopicAlreadyExists = errors.New("topic already registered") - ErrTopicTableMissing = errors.New("events table for topic does not exist") + ErrOrganizationNotFound = errors.New("organization not found") + ErrOrganizationAlreadyExists = errors.New("organization already registered") ) -// topic represents an internal topic with its subscriptions and poll state -type topic struct { - organization string - name TopicName +// organization represents an internal organization with its subscriptions and poll state +type organization struct { + id OrganizationID subscribers []chan<- []Event // Registered subscription channels subscriberMu sync.RWMutex // Polling state - knownVersion string // Last known version from states table + knownVersion string // Last known version from organization_states table lastPolled time.Time // Timestamp of last successful poll } -// topicRegistry manages all registered topics -type topicRegistry struct { - topics map[TopicName]*topic - mu sync.RWMutex +// organizationRegistry manages all registered organizations +type organizationRegistry struct { + orgs map[OrganizationID]*organization + mu sync.RWMutex } -// newTopicRegistry creates a new topic registry -func newTopicRegistry() *topicRegistry { - return &topicRegistry{ - topics: make(map[TopicName]*topic), +// newOrganizationRegistry creates a new organization registry +func newOrganizationRegistry() *organizationRegistry { + return &organizationRegistry{ + orgs: make(map[OrganizationID]*organization), } } -// register adds a new topic to the registry -func (r *topicRegistry) register(organization string, name TopicName) error { +// register adds a new organization to the registry +func (r *organizationRegistry) register(id OrganizationID) error { r.mu.Lock() defer r.mu.Unlock() - if _, exists := r.topics[name]; exists { - return ErrTopicAlreadyExists + if _, exists := r.orgs[id]; exists { + return ErrOrganizationAlreadyExists } - r.topics[name] = &topic{ - organization: organization, - name: name, - subscribers: make([]chan<- []Event, 0), - lastPolled: time.Now(), // Start from now, don't replay old events + r.orgs[id] = &organization{ + id: id, + subscribers: make([]chan<- []Event, 0), + lastPolled: time.Now(), // Start from now, don't replay old events } return nil } -// get retrieves a topic by name -func (r *topicRegistry) get(name TopicName) (*topic, error) { +// get retrieves an organization by ID +func (r *organizationRegistry) get(id OrganizationID) (*organization, error) { r.mu.RLock() defer r.mu.RUnlock() - t, exists := r.topics[name] + org, exists := r.orgs[id] if !exists { - return nil, ErrTopicNotFound + return nil, ErrOrganizationNotFound } - return t, nil + return org, nil } -// addSubscriber adds a subscription channel to a topic -func (r *topicRegistry) addSubscriber(name TopicName, ch chan<- []Event) error { +// addSubscriber adds a subscription channel to an organization +func (r *organizationRegistry) addSubscriber(id OrganizationID, ch chan<- []Event) error { r.mu.RLock() - t, exists := r.topics[name] + org, exists := r.orgs[id] r.mu.RUnlock() if !exists { - return ErrTopicNotFound + return ErrOrganizationNotFound } - t.subscriberMu.Lock() - defer t.subscriberMu.Unlock() - t.subscribers = append(t.subscribers, ch) + org.subscriberMu.Lock() + defer org.subscriberMu.Unlock() + org.subscribers = append(org.subscribers, ch) return nil } -// getAll returns all registered topics -func (r *topicRegistry) getAll() []*topic { +// getAll returns all registered organizations +func (r *organizationRegistry) getAll() []*organization { r.mu.RLock() defer r.mu.RUnlock() - topics := make([]*topic, 0, len(r.topics)) - for _, t := range r.topics { - topics = append(topics, t) + orgs := make([]*organization, 0, len(r.orgs)) + for _, org := range r.orgs { + orgs = append(orgs, org) } - return topics + return orgs } -// updatePollState updates the polling state for a topic -func (t *topic) updatePollState(version string, polledAt time.Time) { - t.knownVersion = version - t.lastPolled = polledAt +// updatePollState updates the polling state for an organization +func (org *organization) updatePollState(version string, polledAt time.Time) { + org.knownVersion = version + org.lastPolled = polledAt } // getSubscribers returns a copy of the subscribers list -func (t *topic) getSubscribers() []chan<- []Event { - t.subscriberMu.RLock() - defer t.subscriberMu.RUnlock() +func (org *organization) getSubscribers() []chan<- []Event { + org.subscriberMu.RLock() + defer org.subscriberMu.RUnlock() - subs := make([]chan<- []Event, len(t.subscribers)) - copy(subs, t.subscribers) + subs := make([]chan<- []Event, len(org.subscribers)) + copy(subs, org.subscribers) return subs } diff --git a/gateway/gateway-controller/pkg/eventhub/types.go b/gateway/gateway-controller/pkg/eventhub/types.go index 2b46bcb85..cf183e5fa 100644 --- a/gateway/gateway-controller/pkg/eventhub/types.go +++ b/gateway/gateway-controller/pkg/eventhub/types.go @@ -5,24 +5,35 @@ import ( "time" ) -// TopicName represents a unique topic identifier -type TopicName string +// OrganizationID represents a unique organization identifier +type OrganizationID string + +// EventType represents the type of event +type EventType string + +// Event type constants +const ( + EventTypeAPI EventType = "API" + EventTypeCertificate EventType = "CERTIFICATE" + EventTypeLLMTemplate EventType = "LLM_TEMPLATE" +) // Event represents a single event in the hub type Event struct { - ID int64 - TopicName TopicName - ProcessedTimestamp time.Time // When event was recorded in DB - OriginatedTimestamp time.Time // When event was created - EventData []byte // JSON serialized payload + OrganizationID OrganizationID // Organization this event belongs to + ProcessedTimestamp time.Time // When event was recorded in DB + OriginatedTimestamp time.Time // When event was created + EventType EventType // Type of event (API, CERTIFICATE, etc.) + Action string // CREATE, UPDATE, or DELETE + EntityID string // ID of the affected entity + EventData []byte // JSON serialized payload } -// TopicState represents the version state for a topic -type TopicState struct { - Organization string - TopicName TopicName +// OrganizationState represents the version state for an organization +type OrganizationState struct { + Organization string // Organization ID VersionID string // UUID that changes on every modification - UpdatedAt time.Time + UpdatedAt time.Time // Last update timestamp } // EventHub is the main interface for the message broker @@ -30,18 +41,19 @@ type EventHub interface { // Initialize sets up database connections and starts background poller Initialize(ctx context.Context) error - // RegisterTopic registers a topic - // Returns error if the events table for this topic does not exist - // Creates entry in States table with empty version - RegisterTopic(organization string, topicName TopicName) error + // RegisterOrganization registers an organization for event tracking + // Creates entry in organization_states table with empty version + RegisterOrganization(organizationID OrganizationID) error - // PublishEvent publishes an event to a topic - // Updates the states table and events table atomically - PublishEvent(ctx context.Context, organization string, topicName TopicName, eventData []byte) error + // PublishEvent publishes an event for an organization + // Updates the organization_states and events tables atomically + PublishEvent(ctx context.Context, organizationID OrganizationID, eventType EventType, + action, entityID string, eventData []byte) error - // RegisterSubscription registers a channel to receive events for a topic + // Subscribe registers a channel to receive events for an organization // Events are delivered as batches (arrays) based on poll cycle - RegisterSubscription(topicName TopicName, eventChan chan<- []Event) error + // Subscriber receives ALL event types and should filter by EventType if needed + Subscribe(organizationID OrganizationID, eventChan chan<- []Event) error // CleanUpEvents removes events between the specified time range CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index 670f1c2ee..30a06d4bb 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -324,6 +324,66 @@ func (s *SQLiteStorage) initSchema() error { version = 8 } + if version == 8 { + // Migrate to organization-centric Event Hub architecture + s.logger.Info("Migrating to organization-centric Event Hub (version 9)") + + // Step 1: Create new organization_states table + if _, err := s.db.Exec(`CREATE TABLE organization_states ( + organization TEXT PRIMARY KEY, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + );`); err != nil { + return fmt.Errorf("failed to create organization_states table: %w", err) + } + + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_organization_states_updated ON organization_states(updated_at);`); err != nil { + return fmt.Errorf("failed to create organization_states index: %w", err) + } + + // Step 2: Migrate state data (consolidate per organization) + if _, err := s.db.Exec(`INSERT INTO organization_states (organization, version_id, updated_at) + SELECT organization, MAX(version_id), MAX(updated_at) + FROM topic_states + GROUP BY organization;`); err != nil { + return fmt.Errorf("failed to migrate state data: %w", err) + } + + // Step 3: Create unified events table + if _, err := s.db.Exec(`CREATE TABLE events ( + organization_id TEXT NOT NULL, + processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + originated_timestamp TIMESTAMP NOT NULL, + event_type TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), + entity_id TEXT NOT NULL, + event_data TEXT NOT NULL, + PRIMARY KEY (organization_id, processed_timestamp) + );`); err != nil { + return fmt.Errorf("failed to create events table: %w", err) + } + + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_events_lookup ON events(organization_id, processed_timestamp);`); err != nil { + return fmt.Errorf("failed to create events lookup index: %w", err) + } + + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type);`); err != nil { + return fmt.Errorf("failed to create events type index: %w", err) + } + + // Step 4: Drop old topic_states table (data already migrated) + if _, err := s.db.Exec(`DROP TABLE topic_states;`); err != nil { + return fmt.Errorf("failed to drop topic_states table: %w", err) + } + + if _, err := s.db.Exec("PRAGMA user_version = 9"); err != nil { + return fmt.Errorf("failed to set schema version to 9: %w", err) + } + + s.logger.Info("Schema migrated to version 9 (organization-centric Event Hub)") + version = 9 + } + s.logger.Info("Database schema up to date", zap.Int("version", version)) } From 34524cbd03dc9cadd4689fb0bba14580d4032283 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 28 Dec 2025 21:27:34 +0530 Subject: [PATCH 04/24] Add Event Listener: Not tested --- .../pkg/eventlistener/api_processor.go | 322 ++++++++++++++++++ .../pkg/eventlistener/listener.go | 145 ++++++++ 2 files changed, 467 insertions(+) create mode 100644 gateway/gateway-controller/pkg/eventlistener/api_processor.go create mode 100644 gateway/gateway-controller/pkg/eventlistener/listener.go diff --git a/gateway/gateway-controller/pkg/eventlistener/api_processor.go b/gateway/gateway-controller/pkg/eventlistener/api_processor.go new file mode 100644 index 000000000..dd8fa2848 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/api_processor.go @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package eventlistener + +import ( + "context" + "fmt" + "strings" + "time" + + api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" + policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" + "go.uber.org/zap" +) + +// processAPIEvents handles API events based on action type +func (el *EventListener) processAPIEvents(event eventhub.Event) { + log := el.logger.With( + zap.String("api_id", event.EntityID), + zap.String("action", event.Action), + ) + + apiID := event.EntityID + + switch event.Action { + case "CREATE", "UPDATE": + el.handleAPICreateOrUpdate(apiID, log) + case "DELETE": + el.handleAPIDelete(apiID, log) + default: + log.Warn("Unknown action type") + } +} + +// handleAPICreateOrUpdate fetches the API from DB and updates XDS +func (el *EventListener) handleAPICreateOrUpdate(apiID string, log *zap.Logger) { + // 1. Fetch API configuration from database + config, err := el.db.GetConfig(apiID) + if err != nil { + log.Error("Failed to fetch API config from database", zap.Error(err)) + return + } + + // 2. Update in-memory store (add or update) + _, err = el.store.Get(apiID) + if err != nil { + // Config doesn't exist in memory - add it + if err := el.store.Add(config); err != nil { + log.Error("Failed to add config to store", zap.Error(err)) + return + } + log.Info("Added API config to in-memory store") + } else { + // Config exists - update it + if err := el.store.Update(config); err != nil { + log.Error("Failed to update config in store", zap.Error(err)) + return + } + log.Info("Updated API config in in-memory store") + } + + var storedPolicy *models.StoredPolicyConfig + + if el.policyManager != nil { + storedPolicy = el.buildStoredPolicyFromAPI(config) + } + + // 3. Trigger async XDS snapshot update + correlationID := fmt.Sprintf("event-%s-%d", apiID, time.Now().UnixNano()) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := el.snapshotManager.UpdateSnapshot(ctx, correlationID); err != nil { + log.Error("Failed to update xDS snapshot", zap.Error(err)) + } else { + log.Info("xDS snapshot updated successfully") + } + }() + + // 4. Update PolicyManager (if configured) + if storedPolicy != nil { + if err := el.policyManager.AddPolicy(storedPolicy); err != nil { + log.Warn("Failed to add policy to PolicyManager", zap.Error(err)) + } else { + log.Info("Added policy to PolicyManager") + } + } +} + +// handleAPIDelete removes the API from in-memory store and updates XDS +func (el *EventListener) handleAPIDelete(apiID string, log *zap.Logger) { + // 1. Check if config exists in store (for logging/policy removal) + config, err := el.store.Get(apiID) + if err != nil { + log.Warn("Config not found in store, may already be deleted") + // Continue anyway to ensure cleanup + } + + // 2. Remove from in-memory store + if err := el.store.Delete(apiID); err != nil { + log.Warn("Failed to delete config from store", zap.Error(err)) + // Continue - config may not exist + } else { + log.Info("Removed API config from in-memory store") + } + + // 3. Trigger async XDS snapshot update + correlationID := fmt.Sprintf("event-delete-%s-%d", apiID, time.Now().UnixNano()) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := el.snapshotManager.UpdateSnapshot(ctx, correlationID); err != nil { + log.Error("Failed to update xDS snapshot after delete", zap.Error(err)) + } else { + log.Info("xDS snapshot updated after delete") + } + }() + + // 4. Remove from PolicyManager (if configured) + if el.policyManager != nil && config != nil { + policyID := apiID + "-policies" + if err := el.policyManager.RemovePolicy(policyID); err != nil { + log.Warn("Failed to remove policy from PolicyManager", zap.Error(err)) + } else { + log.Info("Removed policy from PolicyManager") + } + } +} + +// buildStoredPolicyFromAPI constructs a StoredPolicyConfig from an API config +// Merging rules: When operation has policies, they define the order (can reorder, override, or extend API policies). +// Remaining API-level policies not mentioned in operation policies are appended at the end. +// When operation has no policies, API-level policies are used in their declared order. +// RouteKey uses the fully qualified route path (context + operation path) and must match the route name format +// used by the xDS translator for consistency. +func (el *EventListener) buildStoredPolicyFromAPI(cfg *models.StoredConfig) *models.StoredPolicyConfig { + apiCfg := &cfg.Configuration + + // Collect API-level policies + apiPolicies := make(map[string]policyenginev1.PolicyInstance) // name -> policy + if cfg.GetPolicies() != nil { + for _, p := range *cfg.GetPolicies() { + apiPolicies[p.Name] = convertAPIPolicy(p) + } + } + + routes := make([]policyenginev1.PolicyChain, 0) + switch apiCfg.Kind { + case api.Asyncwebsub: + // Build routes with merged policies for WebSub + apiData, err := apiCfg.Spec.AsWebhookAPIData() + if err != nil { + el.logger.Error("Failed to parse WebSub API data", zap.Error(err)) + return nil + } + for _, ch := range apiData.Channels { + var finalPolicies []policyenginev1.PolicyInstance + + if ch.Policies != nil && len(*ch.Policies) > 0 { + // Operation has policies: use operation policy order as authoritative + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*ch.Policies)) + addedNames := make(map[string]struct{}) + + for _, opPolicy := range *ch.Policies { + finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) + addedNames[opPolicy.Name] = struct{}{} + } + + // Add any API-level policies not mentioned in operation policies (append at end) + if apiData.Policies != nil { + for _, apiPolicy := range *apiData.Policies { + if _, exists := addedNames[apiPolicy.Name]; !exists { + finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) + } + } + } + } else { + // No operation policies: use API-level policies in their declared order + if apiData.Policies != nil { + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) + for _, p := range *apiData.Policies { + finalPolicies = append(finalPolicies, apiPolicies[p.Name]) + } + } + } + + routeKey := xds.GenerateRouteName("POST", apiData.Context, apiData.Version, ch.Path, el.routerConfig.GatewayHost) + routes = append(routes, policyenginev1.PolicyChain{ + RouteKey: routeKey, + Policies: finalPolicies, + }) + } + case api.RestApi: + // Build routes with merged policies for REST API + apiData, err := apiCfg.Spec.AsAPIConfigData() + if err != nil { + el.logger.Error("Failed to parse REST API data", zap.Error(err)) + return nil + } + for _, op := range apiData.Operations { + var finalPolicies []policyenginev1.PolicyInstance + + if op.Policies != nil && len(*op.Policies) > 0 { + // Operation has policies: use operation policy order as authoritative + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*op.Policies)) + addedNames := make(map[string]struct{}) + + for _, opPolicy := range *op.Policies { + finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) + addedNames[opPolicy.Name] = struct{}{} + } + + // Add any API-level policies not mentioned in operation policies (append at end) + if apiData.Policies != nil { + for _, apiPolicy := range *apiData.Policies { + if _, exists := addedNames[apiPolicy.Name]; !exists { + finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) + } + } + } + } else { + // No operation policies: use API-level policies in their declared order + if apiData.Policies != nil { + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) + for _, p := range *apiData.Policies { + finalPolicies = append(finalPolicies, apiPolicies[p.Name]) + } + } + } + + // Determine effective vhosts (fallback to global router defaults when not provided) + effectiveMainVHost := el.routerConfig.VHosts.Main.Default + effectiveSandboxVHost := el.routerConfig.VHosts.Sandbox.Default + if apiData.Vhosts != nil { + if strings.TrimSpace(apiData.Vhosts.Main) != "" { + effectiveMainVHost = apiData.Vhosts.Main + } + if apiData.Vhosts.Sandbox != nil && strings.TrimSpace(*apiData.Vhosts.Sandbox) != "" { + effectiveSandboxVHost = *apiData.Vhosts.Sandbox + } + } + + vhosts := []string{effectiveMainVHost} + if apiData.Upstream.Sandbox != nil && apiData.Upstream.Sandbox.Url != nil && + strings.TrimSpace(*apiData.Upstream.Sandbox.Url) != "" { + vhosts = append(vhosts, effectiveSandboxVHost) + } + + for _, vhost := range vhosts { + routes = append(routes, policyenginev1.PolicyChain{ + RouteKey: xds.GenerateRouteName(string(op.Method), apiData.Context, apiData.Version, op.Path, vhost), + Policies: finalPolicies, + }) + } + } + } + + // If there are no policies at all, return nil (skip creation) + policyCount := 0 + for _, r := range routes { + policyCount += len(r.Policies) + } + if policyCount == 0 { + return nil + } + + now := time.Now().Unix() + stored := &models.StoredPolicyConfig{ + ID: cfg.ID + "-policies", + Configuration: policyenginev1.Configuration{ + Routes: routes, + Metadata: policyenginev1.Metadata{ + CreatedAt: now, + UpdatedAt: now, + ResourceVersion: 0, + APIName: cfg.GetDisplayName(), + Version: cfg.GetVersion(), + Context: cfg.GetContext(), + }, + }, + Version: 0, + } + return stored +} + +// convertAPIPolicy converts generated api.Policy to policyenginev1.PolicyInstance +func convertAPIPolicy(p api.Policy) policyenginev1.PolicyInstance { + paramsMap := make(map[string]interface{}) + if p.Params != nil { + for k, v := range *p.Params { + paramsMap[k] = v + } + } + return policyenginev1.PolicyInstance{ + Name: p.Name, + Version: p.Version, + Enabled: true, // Default to enabled + ExecutionCondition: p.ExecutionCondition, + Parameters: paramsMap, + } +} diff --git a/gateway/gateway-controller/pkg/eventlistener/listener.go b/gateway/gateway-controller/pkg/eventlistener/listener.go new file mode 100644 index 000000000..910480146 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/listener.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package eventlistener + +import ( + "context" + "fmt" + "sync" + + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" + "go.uber.org/zap" +) + +// EventListener subscribes to EventHub and processes events to update XDS +type EventListener struct { + eventHub eventhub.EventHub + store *storage.ConfigStore // In-memory config store + db storage.Storage // Persistent storage (SQLite) + snapshotManager *xds.SnapshotManager // XDS snapshot manager + policyManager *policyxds.PolicyManager // Optional: policy manager + routerConfig *config.RouterConfig // Router configuration for vhosts + logger *zap.Logger + + eventChan chan []eventhub.Event // Buffered channel (size 10) + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewEventListener creates a new EventListener instance +func NewEventListener( + eventHub eventhub.EventHub, + store *storage.ConfigStore, + db storage.Storage, + snapshotManager *xds.SnapshotManager, + policyManager *policyxds.PolicyManager, // Can be nil + routerConfig *config.RouterConfig, + logger *zap.Logger, +) *EventListener { + return &EventListener{ + eventHub: eventHub, + store: store, + db: db, + snapshotManager: snapshotManager, + policyManager: policyManager, + routerConfig: routerConfig, + logger: logger, + } +} + +// Start initializes the event listener and starts processing events +func (el *EventListener) Start(ctx context.Context) error { + el.ctx, el.cancel = context.WithCancel(ctx) + + // Create buffered channel with size 10 + el.eventChan = make(chan []eventhub.Event, 10) + + // Register "default" organization (idempotent - may already exist) + orgID := eventhub.OrganizationID("default") + if err := el.eventHub.RegisterOrganization(orgID); err != nil { + // Ignore if already exists + el.logger.Debug("Organization may already be registered", zap.String("organization", string(orgID))) + } + + // Subscribe to events + if err := el.eventHub.Subscribe(orgID, el.eventChan); err != nil { + return fmt.Errorf("failed to subscribe to events: %w", err) + } + + // Start processing goroutine + el.wg.Add(1) + // TODO: (VirajSalaka) Should recover in case of panics + go el.processEvents() + + el.logger.Info("EventListener started", zap.String("organization", "default")) + return nil +} + +// processEvents is a goroutine that continuously processes events from the channel +func (el *EventListener) processEvents() { + defer el.wg.Done() + + for { + select { + case <-el.ctx.Done(): + el.logger.Info("EventListener stopping") + return + + case events := <-el.eventChan: + for _, event := range events { + el.handleEvent(event) + } + } + } +} + +// handleEvent processes a single event and delegates based on event type +func (el *EventListener) handleEvent(event eventhub.Event) { + log := el.logger.With( + zap.String("event_type", string(event.EventType)), + zap.String("action", event.Action), + zap.String("entity_id", event.EntityID), + ) + + switch event.EventType { + case eventhub.EventTypeAPI: + el.processAPIEvents(event) + default: + log.Debug("Ignoring non-API event") + } +} + +// Stop gracefully shuts down the event listener +func (el *EventListener) Stop() { + if el.cancel != nil { + el.cancel() + } + el.wg.Wait() + + if el.eventChan != nil { + close(el.eventChan) + } + + el.logger.Info("EventListener stopped") +} From 7e4b8fdafdc7df75afee28d84a4e6335da8ec589 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 28 Dec 2025 23:21:16 +0530 Subject: [PATCH 05/24] Make Event Listener Generic: Not tested --- .../pkg/eventlistener/api_processor.go | 6 +- .../pkg/eventlistener/event_source.go | 98 ++++++++ .../pkg/eventlistener/eventhub_adapter.go | 187 ++++++++++++++++ .../pkg/eventlistener/listener.go | 60 ++--- .../pkg/eventlistener/mock_event_source.go | 211 ++++++++++++++++++ 5 files changed, 533 insertions(+), 29 deletions(-) create mode 100644 gateway/gateway-controller/pkg/eventlistener/event_source.go create mode 100644 gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go create mode 100644 gateway/gateway-controller/pkg/eventlistener/mock_event_source.go diff --git a/gateway/gateway-controller/pkg/eventlistener/api_processor.go b/gateway/gateway-controller/pkg/eventlistener/api_processor.go index dd8fa2848..74b65a8f4 100644 --- a/gateway/gateway-controller/pkg/eventlistener/api_processor.go +++ b/gateway/gateway-controller/pkg/eventlistener/api_processor.go @@ -25,15 +25,15 @@ import ( "time" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" "go.uber.org/zap" ) -// processAPIEvents handles API events based on action type -func (el *EventListener) processAPIEvents(event eventhub.Event) { +// processAPIEvents handles API events based on action type. +// Works with the generic Event type from any EventSource implementation. +func (el *EventListener) processAPIEvents(event Event) { log := el.logger.With( zap.String("api_id", event.EntityID), zap.String("action", event.Action), diff --git a/gateway/gateway-controller/pkg/eventlistener/event_source.go b/gateway/gateway-controller/pkg/eventlistener/event_source.go new file mode 100644 index 000000000..269214a5d --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/event_source.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package eventlistener + +import ( + "context" + "time" +) + +// Event represents a generic event from any event source. +// This is a simplified, agnostic event structure that can be populated +// by different event source implementations (EventHub, Kafka, RabbitMQ, etc.) +type Event struct { + // OrganizationID identifies which organization this event belongs to + OrganizationID string + + // EventType describes the kind of event (e.g., "API", "CERTIFICATE", "LLM_TEMPLATE") + EventType string + + // Action describes what happened (e.g., "CREATE", "UPDATE", "DELETE") + Action string + + // EntityID identifies the specific entity affected by the event + EntityID string + + // EventData contains the serialized event payload (typically JSON) + EventData []byte + + // Timestamp indicates when the event occurred + Timestamp time.Time +} + +// EventSource defines the interface for any event delivery mechanism. +// Implementations can use EventHub, message brokers (Kafka, RabbitMQ, NATS), +// or any other pub/sub system. +// +// Design principles: +// - Simple and focused: only what EventListener needs +// - Technology agnostic: no assumptions about the underlying system +// - Testable: easy to mock for unit tests +type EventSource interface { + // Subscribe registers to receive events for a specific organization. + // Events are delivered as batches via the provided channel. + // + // Parameters: + // - ctx: Context for cancellation and timeout + // - organizationID: The organization to subscribe to + // - eventChan: Channel where event batches will be sent + // + // Returns: + // - error if subscription fails + // + // Notes: + // - The implementation is responsible for managing the subscription lifecycle + // - Events should be delivered until Unsubscribe is called or ctx is cancelled + // - The channel should NOT be closed by the EventSource + Subscribe(ctx context.Context, organizationID string, eventChan chan<- []Event) error + + // Unsubscribe stops receiving events for a specific organization. + // + // Parameters: + // - organizationID: The organization to unsubscribe from + // + // Returns: + // - error if unsubscribe fails + // + // Notes: + // - After calling Unsubscribe, no more events should be sent to the channel + // - It's safe to call Unsubscribe multiple times + Unsubscribe(organizationID string) error + + // Close gracefully shuts down the event source and cleans up resources. + // + // Returns: + // - error if shutdown fails + // + // Notes: + // - Should unsubscribe all active subscriptions + // - Should wait for in-flight events to be delivered + // - Should be idempotent (safe to call multiple times) + Close() error +} diff --git a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go new file mode 100644 index 000000000..e5d20a89d --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package eventlistener + +import ( + "context" + "fmt" + + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" + "go.uber.org/zap" +) + +// EventHubAdapter adapts the eventhub.EventHub interface to the generic EventSource interface. +// This allows EventListener to work with the existing EventHub implementation while maintaining +// abstraction and testability. +type EventHubAdapter struct { + eventHub eventhub.EventHub + logger *zap.Logger + + // activeSubscriptions tracks which organizations have active subscriptions + // This is used to ensure proper cleanup and prevent duplicate subscriptions + activeSubscriptions map[string]chan<- []Event +} + +// NewEventHubAdapter creates a new adapter that wraps an EventHub instance. +// +// Parameters: +// - eventHub: The EventHub instance to wrap +// - logger: Logger for debugging and error reporting +// +// Returns: +// - EventSource implementation backed by EventHub +func NewEventHubAdapter(eventHub eventhub.EventHub, logger *zap.Logger) EventSource { + return &EventHubAdapter{ + eventHub: eventHub, + logger: logger, + activeSubscriptions: make(map[string]chan<- []Event), + } +} + +// Subscribe implements EventSource.Subscribe by delegating to EventHub. +// It handles organization registration and event conversion. +func (a *EventHubAdapter) Subscribe(ctx context.Context, organizationID string, eventChan chan<- []Event) error { + // Check if already subscribed + if _, exists := a.activeSubscriptions[organizationID]; exists { + return fmt.Errorf("already subscribed to organization: %s", organizationID) + } + + orgID := eventhub.OrganizationID(organizationID) + + // Register organization with EventHub (idempotent operation) + if err := a.eventHub.RegisterOrganization(orgID); err != nil { + a.logger.Debug("Organization may already be registered", + zap.String("organization", organizationID), + zap.Error(err), + ) + // Continue - registration errors are usually not fatal + } + + // Create a bridge channel that receives eventhub.Event and converts to generic Event + bridgeChan := make(chan []eventhub.Event, 10) + + // Subscribe to EventHub + if err := a.eventHub.Subscribe(orgID, bridgeChan); err != nil { + close(bridgeChan) + return fmt.Errorf("failed to subscribe to eventhub: %w", err) + } + + // Track this subscription + a.activeSubscriptions[organizationID] = eventChan + + // Start goroutine to convert and forward events + go a.bridgeEvents(ctx, organizationID, bridgeChan, eventChan) + + a.logger.Info("Subscribed to event source", + zap.String("organization", organizationID), + zap.String("source", "eventhub"), + ) + + return nil +} + +// bridgeEvents converts eventhub.Event to generic Event and forwards to the listener. +// This goroutine runs until the context is cancelled or the bridge channel is closed. +func (a *EventHubAdapter) bridgeEvents( + ctx context.Context, + organizationID string, + from <-chan []eventhub.Event, + to chan<- []Event, +) { + defer func() { + // Clean up subscription tracking + delete(a.activeSubscriptions, organizationID) + a.logger.Debug("Bridge goroutine exiting", + zap.String("organization", organizationID), + ) + }() + + for { + select { + case <-ctx.Done(): + return + + case hubEvents, ok := <-from: + if !ok { + // EventHub channel closed + a.logger.Warn("EventHub channel closed unexpectedly", + zap.String("organization", organizationID), + ) + return + } + + // Convert eventhub.Event to generic Event + genericEvents := make([]Event, len(hubEvents)) + for i, hubEvent := range hubEvents { + genericEvents[i] = Event{ + OrganizationID: string(hubEvent.OrganizationID), + EventType: string(hubEvent.EventType), + Action: hubEvent.Action, + EntityID: hubEvent.EntityID, + EventData: hubEvent.EventData, + Timestamp: hubEvent.ProcessedTimestamp, + } + } + + // Forward to listener + select { + case to <- genericEvents: + // Successfully forwarded + case <-ctx.Done(): + return + } + } + } +} + +// Unsubscribe implements EventSource.Unsubscribe. +// Note: The current EventHub implementation doesn't have an explicit unsubscribe method, +// so we just stop the bridge goroutine by removing the subscription tracking. +func (a *EventHubAdapter) Unsubscribe(organizationID string) error { + if _, exists := a.activeSubscriptions[organizationID]; !exists { + // Not subscribed - this is fine, make it idempotent + return nil + } + + // Remove from tracking - the bridge goroutine will detect context cancellation + // when the listener stops + delete(a.activeSubscriptions, organizationID) + + a.logger.Info("Unsubscribed from event source", + zap.String("organization", organizationID), + ) + + return nil +} + +// Close implements EventSource.Close by delegating to EventHub.Close. +func (a *EventHubAdapter) Close() error { + // Clean up all subscriptions + for orgID := range a.activeSubscriptions { + _ = a.Unsubscribe(orgID) + } + + // Close the underlying EventHub + if err := a.eventHub.Close(); err != nil { + return fmt.Errorf("failed to close eventhub: %w", err) + } + + a.logger.Info("Event source closed", zap.String("source", "eventhub")) + return nil +} diff --git a/gateway/gateway-controller/pkg/eventlistener/listener.go b/gateway/gateway-controller/pkg/eventlistener/listener.go index 910480146..97d3041a7 100644 --- a/gateway/gateway-controller/pkg/eventlistener/listener.go +++ b/gateway/gateway-controller/pkg/eventlistener/listener.go @@ -20,20 +20,20 @@ package eventlistener import ( "context" - "fmt" "sync" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" "go.uber.org/zap" ) -// EventListener subscribes to EventHub and processes events to update XDS +// EventListener subscribes to an EventSource and processes events to update XDS. +// It uses the generic EventSource interface, allowing it to work with different +// event delivery mechanisms (EventHub, Kafka, RabbitMQ, etc.) and enabling easy mocking for tests. type EventListener struct { - eventHub eventhub.EventHub + eventSource EventSource // Generic event source (EventHub, Kafka, etc.) store *storage.ConfigStore // In-memory config store db storage.Storage // Persistent storage (SQLite) snapshotManager *xds.SnapshotManager // XDS snapshot manager @@ -41,15 +41,27 @@ type EventListener struct { routerConfig *config.RouterConfig // Router configuration for vhosts logger *zap.Logger - eventChan chan []eventhub.Event // Buffered channel (size 10) + eventChan chan []Event // Buffered channel (size 10) for generic events ctx context.Context cancel context.CancelFunc wg sync.WaitGroup } -// NewEventListener creates a new EventListener instance +// NewEventListener creates a new EventListener instance. +// +// Parameters: +// - eventSource: The event source to subscribe to (can be EventHubAdapter, MockEventSource, or any EventSource implementation) +// - store: In-memory configuration store +// - db: Persistent storage (SQLite) +// - snapshotManager: xDS snapshot manager for updating Envoy configuration +// - policyManager: Optional policy manager (can be nil) +// - routerConfig: Router configuration for vhosts +// - logger: Structured logger +// +// Returns: +// - *EventListener ready to be started func NewEventListener( - eventHub eventhub.EventHub, + eventSource EventSource, store *storage.ConfigStore, db storage.Storage, snapshotManager *xds.SnapshotManager, @@ -58,7 +70,7 @@ func NewEventListener( logger *zap.Logger, ) *EventListener { return &EventListener{ - eventHub: eventHub, + eventSource: eventSource, store: store, db: db, snapshotManager: snapshotManager, @@ -68,23 +80,18 @@ func NewEventListener( } } -// Start initializes the event listener and starts processing events +// Start initializes the event listener and starts processing events. +// Subscribes to the "default" organization and starts the event processing goroutine. func (el *EventListener) Start(ctx context.Context) error { el.ctx, el.cancel = context.WithCancel(ctx) - // Create buffered channel with size 10 - el.eventChan = make(chan []eventhub.Event, 10) + // Create buffered channel with size 10 for generic events + el.eventChan = make(chan []Event, 10) - // Register "default" organization (idempotent - may already exist) - orgID := eventhub.OrganizationID("default") - if err := el.eventHub.RegisterOrganization(orgID); err != nil { - // Ignore if already exists - el.logger.Debug("Organization may already be registered", zap.String("organization", string(orgID))) - } - - // Subscribe to events - if err := el.eventHub.Subscribe(orgID, el.eventChan); err != nil { - return fmt.Errorf("failed to subscribe to events: %w", err) + // Subscribe to "default" organization events via the EventSource + organizationID := "default" + if err := el.eventSource.Subscribe(ctx, organizationID, el.eventChan); err != nil { + return err // Let the EventSource adapter handle details } // Start processing goroutine @@ -92,7 +99,7 @@ func (el *EventListener) Start(ctx context.Context) error { // TODO: (VirajSalaka) Should recover in case of panics go el.processEvents() - el.logger.Info("EventListener started", zap.String("organization", "default")) + el.logger.Info("EventListener started", zap.String("organization", organizationID)) return nil } @@ -114,16 +121,17 @@ func (el *EventListener) processEvents() { } } -// handleEvent processes a single event and delegates based on event type -func (el *EventListener) handleEvent(event eventhub.Event) { +// handleEvent processes a single event and delegates based on event type. +// Uses the generic Event type that works with any EventSource implementation. +func (el *EventListener) handleEvent(event Event) { log := el.logger.With( - zap.String("event_type", string(event.EventType)), + zap.String("event_type", event.EventType), zap.String("action", event.Action), zap.String("entity_id", event.EntityID), ) switch event.EventType { - case eventhub.EventTypeAPI: + case "API": // EventTypeAPI constant el.processAPIEvents(event) default: log.Debug("Ignoring non-API event") diff --git a/gateway/gateway-controller/pkg/eventlistener/mock_event_source.go b/gateway/gateway-controller/pkg/eventlistener/mock_event_source.go new file mode 100644 index 000000000..4e326c4b2 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/mock_event_source.go @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package eventlistener + +import ( + "context" + "fmt" + "sync" +) + +// MockEventSource is a mock implementation of EventSource for testing. +// It allows tests to control event delivery and verify subscription behavior. +// +// Usage example: +// +// mock := NewMockEventSource() +// listener := NewEventListener(mock, store, db, ...) +// listener.Start(ctx) +// +// // Publish test events +// mock.PublishEvent("default", Event{...}) +// +// // Verify subscription +// if !mock.IsSubscribed("default") { +// t.Error("Expected subscription to default org") +// } +type MockEventSource struct { + mu sync.RWMutex + subscriptions map[string]chan<- []Event + closed bool + + // PublishedEvents tracks all events published through this mock + PublishedEvents []Event + + // SubscribeCalls tracks Subscribe method invocations + SubscribeCalls []SubscribeCall + + // UnsubscribeCalls tracks Unsubscribe method invocations + UnsubscribeCalls []string + + // Errors can be set to simulate failures + SubscribeError error + UnsubscribeError error + CloseError error +} + +// SubscribeCall records a Subscribe method invocation +type SubscribeCall struct { + OrganizationID string + Context context.Context +} + +// NewMockEventSource creates a new mock event source for testing. +func NewMockEventSource() *MockEventSource { + return &MockEventSource{ + subscriptions: make(map[string]chan<- []Event), + PublishedEvents: make([]Event, 0), + SubscribeCalls: make([]SubscribeCall, 0), + UnsubscribeCalls: make([]string, 0), + } +} + +// Subscribe implements EventSource.Subscribe for testing. +// Records the call and sets up event delivery. +func (m *MockEventSource) Subscribe(ctx context.Context, organizationID string, eventChan chan<- []Event) error { + m.mu.Lock() + defer m.mu.Unlock() + + // Record the call + m.SubscribeCalls = append(m.SubscribeCalls, SubscribeCall{ + OrganizationID: organizationID, + Context: ctx, + }) + + // Return configured error if set + if m.SubscribeError != nil { + return m.SubscribeError + } + + // Check if already subscribed + if _, exists := m.subscriptions[organizationID]; exists { + return fmt.Errorf("already subscribed to organization: %s", organizationID) + } + + // Store subscription + m.subscriptions[organizationID] = eventChan + + return nil +} + +// Unsubscribe implements EventSource.Unsubscribe for testing. +// Records the call and removes the subscription. +func (m *MockEventSource) Unsubscribe(organizationID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + // Record the call + m.UnsubscribeCalls = append(m.UnsubscribeCalls, organizationID) + + // Return configured error if set + if m.UnsubscribeError != nil { + return m.UnsubscribeError + } + + // Remove subscription + delete(m.subscriptions, organizationID) + + return nil +} + +// Close implements EventSource.Close for testing. +func (m *MockEventSource) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + + // Return configured error if set + if m.CloseError != nil { + return m.CloseError + } + + m.closed = true + m.subscriptions = make(map[string]chan<- []Event) + + return nil +} + +// PublishEvent publishes an event to subscribers (test helper). +// This simulates an event being delivered from the event source. +// +// Parameters: +// - organizationID: The organization to publish to +// - events: One or more events to publish as a batch +// +// Returns: +// - error if no subscription exists for the organization +func (m *MockEventSource) PublishEvent(organizationID string, events ...Event) error { + m.mu.RLock() + eventChan, exists := m.subscriptions[organizationID] + m.mu.RUnlock() + + if !exists { + return fmt.Errorf("no subscription for organization: %s", organizationID) + } + + // Track published events + m.mu.Lock() + m.PublishedEvents = append(m.PublishedEvents, events...) + m.mu.Unlock() + + // Send events to subscriber + eventChan <- events + + return nil +} + +// IsSubscribed checks if an organization has an active subscription (test helper). +func (m *MockEventSource) IsSubscribed(organizationID string) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + _, exists := m.subscriptions[organizationID] + return exists +} + +// IsClosed checks if Close has been called (test helper). +func (m *MockEventSource) IsClosed() bool { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.closed +} + +// Reset clears all recorded calls and state (test helper). +// Useful for resetting state between test cases. +func (m *MockEventSource) Reset() { + m.mu.Lock() + defer m.mu.Unlock() + + m.subscriptions = make(map[string]chan<- []Event) + m.closed = false + m.PublishedEvents = make([]Event, 0) + m.SubscribeCalls = make([]SubscribeCall, 0) + m.UnsubscribeCalls = make([]string, 0) + m.SubscribeError = nil + m.UnsubscribeError = nil + m.CloseError = nil +} + +// GetSubscriptionCount returns the number of active subscriptions (test helper). +func (m *MockEventSource) GetSubscriptionCount() int { + m.mu.RLock() + defer m.mu.RUnlock() + + return len(m.subscriptions) +} From bae76cfdaa1856bed7f5cc1f08f4a0183ef3fe90 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 31 Dec 2025 09:21:09 +0530 Subject: [PATCH 06/24] Example Usage for event hub --- .../pkg/eventlistener/EXAMPLE_USAGE.md | 381 ++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md diff --git a/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md b/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md new file mode 100644 index 000000000..c93e01642 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md @@ -0,0 +1,381 @@ +# EventListener Usage Examples + +This document demonstrates how to use the EventListener with different EventSource implementations. + +## Architecture Overview + +The EventListener is designed with a generic event source abstraction: + +``` +┌─────────────────┐ +│ EventListener │ +└────────┬────────┘ + │ depends on + ▼ +┌─────────────────┐ +│ EventSource │ (interface) +└────────┬────────┘ + │ implemented by + ├─────────────────────┐ + ▼ ▼ +┌──────────────────┐ ┌────────────────┐ +│ EventHubAdapter │ │ MockEventSource│ +└──────────────────┘ └────────────────┘ + │ │ + ▼ ▼ +┌──────────────────┐ ┌────────────────┐ +│ EventHub │ │ Test Suite │ +│ (Database-based) │ │ (In-memory) │ +└──────────────────┘ └────────────────┘ +``` + +## Example 1: Using with EventHub (Production) + +```go +package main + +import ( + "context" + "database/sql" + "time" + + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" + "go.uber.org/zap" +) + +func main() { + logger, _ := zap.NewProduction() + defer logger.Sync() + + // Initialize dependencies + db := initializeDatabase() // *sql.DB + store := storage.NewConfigStore() + sqliteStorage := storage.NewSQLiteStorage(db, logger) + snapshotManager := xds.NewSnapshotManager(store, logger) + policyManager := policyxds.NewPolicyManager(logger) + routerConfig := loadRouterConfig() + + // Create EventHub + hubConfig := eventhub.DefaultConfig() + eventHub := eventhub.New(db, logger, hubConfig) + + // Initialize EventHub + ctx := context.Background() + if err := eventHub.Initialize(ctx); err != nil { + logger.Fatal("Failed to initialize EventHub", zap.Error(err)) + } + + // Wrap EventHub with adapter + eventSource := eventlistener.NewEventHubAdapter(eventHub, logger) + + // Create EventListener with the adapter + listener := eventlistener.NewEventListener( + eventSource, + store, + sqliteStorage, + snapshotManager, + policyManager, + routerConfig, + logger, + ) + + // Start listening for events + if err := listener.Start(ctx); err != nil { + logger.Fatal("Failed to start EventListener", zap.Error(err)) + } + + logger.Info("EventListener is now running") + + // ... application runs ... + + // Graceful shutdown + listener.Stop() + eventSource.Close() +} + +func initializeDatabase() *sql.DB { + // Implementation details... + return nil +} + +func loadRouterConfig() *config.RouterConfig { + // Implementation details... + return nil +} +``` + +## Example 2: Using MockEventSource for Testing + +```go +package mypackage + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" + "go.uber.org/zap" +) + +func TestEventListenerProcessesAPIEvents(t *testing.T) { + logger := zap.NewNop() + + // Create mock event source + mockSource := eventlistener.NewMockEventSource() + + // Setup test dependencies + store := setupTestStore() + db := setupTestDatabase() + snapshotManager := setupTestSnapshotManager() + routerConfig := setupTestRouterConfig() + + // Create EventListener with mock + listener := eventlistener.NewEventListener( + mockSource, + store, + db, + snapshotManager, + nil, // no policy manager for this test + routerConfig, + logger, + ) + + // Start listener + ctx := context.Background() + err := listener.Start(ctx) + assert.NoError(t, err) + + // Verify subscription was created + assert.True(t, mockSource.IsSubscribed("default")) + assert.Equal(t, 1, len(mockSource.SubscribeCalls)) + + // Publish test event + testEvent := eventlistener.Event{ + OrganizationID: "default", + EventType: "API", + Action: "CREATE", + EntityID: "test-api-123", + EventData: []byte(`{"name": "test"}`), + Timestamp: time.Now(), + } + + err = mockSource.PublishEvent("default", testEvent) + assert.NoError(t, err) + + // Allow time for processing + time.Sleep(100 * time.Millisecond) + + // Verify the event was processed + // (Check side effects in store, snapshotManager, etc.) + + // Cleanup + listener.Stop() + assert.True(t, mockSource.IsClosed()) +} + +func TestEventListenerHandlesSubscriptionError(t *testing.T) { + logger := zap.NewNop() + mockSource := eventlistener.NewMockEventSource() + + // Configure mock to return error + mockSource.SubscribeError = assert.AnError + + listener := eventlistener.NewEventListener( + mockSource, + setupTestStore(), + setupTestDatabase(), + setupTestSnapshotManager(), + nil, + setupTestRouterConfig(), + logger, + ) + + // Start should fail + ctx := context.Background() + err := listener.Start(ctx) + assert.Error(t, err) +} + +// Test helper functions +func setupTestStore() *storage.ConfigStore { /* ... */ return nil } +func setupTestDatabase() storage.Storage { /* ... */ return nil } +func setupTestSnapshotManager() *xds.SnapshotManager { /* ... */ return nil } +func setupTestRouterConfig() *config.RouterConfig { /* ... */ return nil } +``` + +## Example 3: Implementing a Custom EventSource (Kafka) + +You can implement a Kafka-based event source: + +```go +package kafka + +import ( + "context" + "encoding/json" + "sync" + + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +// KafkaEventSource implements EventSource using Apache Kafka +type KafkaEventSource struct { + brokers []string + logger *zap.Logger + + mu sync.RWMutex + readers map[string]*kafka.Reader + subscriptions map[string]chan<- []eventlistener.Event +} + +func NewKafkaEventSource(brokers []string, logger *zap.Logger) eventlistener.EventSource { + return &KafkaEventSource{ + brokers: brokers, + logger: logger, + readers: make(map[string]*kafka.Reader), + subscriptions: make(map[string]chan<- []eventlistener.Event), + } +} + +func (k *KafkaEventSource) Subscribe(ctx context.Context, organizationID string, eventChan chan<- []eventlistener.Event) error { + k.mu.Lock() + defer k.mu.Unlock() + + // Create Kafka reader for the organization's topic + topic := "gateway-events-" + organizationID + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: k.brokers, + Topic: topic, + GroupID: "gateway-controller", + }) + + k.readers[organizationID] = reader + k.subscriptions[organizationID] = eventChan + + // Start consuming goroutine + go k.consumeEvents(ctx, organizationID, reader, eventChan) + + k.logger.Info("Subscribed to Kafka topic", + zap.String("organization", organizationID), + zap.String("topic", topic), + ) + + return nil +} + +func (k *KafkaEventSource) consumeEvents( + ctx context.Context, + organizationID string, + reader *kafka.Reader, + eventChan chan<- []eventlistener.Event, +) { + for { + msg, err := reader.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + return // Context cancelled + } + k.logger.Error("Failed to fetch Kafka message", zap.Error(err)) + continue + } + + // Parse event from Kafka message + var event eventlistener.Event + if err := json.Unmarshal(msg.Value, &event); err != nil { + k.logger.Error("Failed to unmarshal event", zap.Error(err)) + continue + } + + // Send as batch (single event) + select { + case eventChan <- []eventlistener.Event{event}: + // Successfully sent + if err := reader.CommitMessages(ctx, msg); err != nil { + k.logger.Error("Failed to commit Kafka message", zap.Error(err)) + } + case <-ctx.Done(): + return + } + } +} + +func (k *KafkaEventSource) Unsubscribe(organizationID string) error { + k.mu.Lock() + defer k.mu.Unlock() + + if reader, exists := k.readers[organizationID]; exists { + reader.Close() + delete(k.readers, organizationID) + delete(k.subscriptions, organizationID) + } + + return nil +} + +func (k *KafkaEventSource) Close() error { + k.mu.Lock() + defer k.mu.Unlock() + + for orgID, reader := range k.readers { + reader.Close() + delete(k.readers, orgID) + } + + k.subscriptions = make(map[string]chan<- []eventlistener.Event) + return nil +} +``` + +Then use it like this: + +```go +// Create Kafka event source +kafkaSource := kafka.NewKafkaEventSource( + []string{"localhost:9092"}, + logger, +) + +// Use with EventListener +listener := eventlistener.NewEventListener( + kafkaSource, + store, + db, + snapshotManager, + policyManager, + routerConfig, + logger, +) +``` + +## Benefits of this Architecture + +1. **Testability**: Easy to test EventListener with MockEventSource +2. **Flexibility**: Switch between EventHub, Kafka, RabbitMQ, etc. +3. **Decoupling**: EventListener doesn't depend on specific implementation details +4. **Maintainability**: Clear interfaces and responsibilities +5. **Extensibility**: Easy to add new event source implementations + +## Event Flow + +``` +EventSource → Subscribe(orgID, chan) → EventListener + ↓ ↓ +Publishes events Receives []Event + ↓ ↓ +Forward to channel processEvents() + ↓ ↓ +Event batches handleEvent() + ↓ + processAPIEvents() + ↓ + Update XDS & Policies +``` From 9f22d94abc5bfd890dc7e2dcb5ab766e782f51a8 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 31 Dec 2025 09:22:50 +0530 Subject: [PATCH 07/24] Fix build failure --- gateway/gateway-controller/pkg/policyxds/snapshot.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gateway/gateway-controller/pkg/policyxds/snapshot.go b/gateway/gateway-controller/pkg/policyxds/snapshot.go index 4e27d86e6..11a0474b9 100644 --- a/gateway/gateway-controller/pkg/policyxds/snapshot.go +++ b/gateway/gateway-controller/pkg/policyxds/snapshot.go @@ -29,6 +29,8 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "go.uber.org/zap" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" ) // SnapshotManager manages xDS snapshots for policy configurations From 7672accdabfc8f4450755c9cecc3e1f2a9f358bb Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 31 Dec 2025 16:47:26 +0530 Subject: [PATCH 08/24] Generalize Eventhub Implementation --- .../pkg/eventhub/backend.go | 92 ++++ .../pkg/eventhub/eventhub.go | 158 ++----- .../gateway-controller/pkg/eventhub/poller.go | 150 ------- .../pkg/eventhub/sqlite_backend.go | 417 ++++++++++++++++++ .../gateway-controller/pkg/eventhub/store.go | 244 ---------- .../gateway-controller/pkg/eventhub/topic.go | 23 + 6 files changed, 565 insertions(+), 519 deletions(-) create mode 100644 gateway/gateway-controller/pkg/eventhub/backend.go delete mode 100644 gateway/gateway-controller/pkg/eventhub/poller.go create mode 100644 gateway/gateway-controller/pkg/eventhub/sqlite_backend.go delete mode 100644 gateway/gateway-controller/pkg/eventhub/store.go diff --git a/gateway/gateway-controller/pkg/eventhub/backend.go b/gateway/gateway-controller/pkg/eventhub/backend.go new file mode 100644 index 000000000..94afc3167 --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/backend.go @@ -0,0 +1,92 @@ +package eventhub + +import ( + "context" + "time" +) + +// EventhubImpl is the interface that different message broker implementations must satisfy. +// Implementations can include SQLite (polling-based), NATS, Azure Service Bus, Kafka, etc. +type EventhubImpl interface { + // Initialize sets up the backend connection and resources. + // For SQLite: opens database, creates tables + // For NATS: connects to server, sets up streams + // For Azure Service Bus: connects to namespace, creates topics + Initialize(ctx context.Context) error + + // RegisterOrganization creates the necessary resources for tracking an organization. + // For SQLite: creates entry in organization_states table + // For NATS: creates subject/stream for the organization + // For Azure Service Bus: creates topic for the organization + RegisterOrganization(ctx context.Context, orgID OrganizationID) error + + // Publish publishes an event for an organization. + // The implementation should ensure delivery semantics appropriate for the broker. + Publish(ctx context.Context, orgID OrganizationID, eventType EventType, + action, entityID string, eventData []byte) error + + // Subscribe registers a channel to receive events for an organization. + // Events are delivered as batches (slices) when available. + // The subscriber receives ALL event types and should filter if needed. + Subscribe(orgID OrganizationID, eventChan chan<- []Event) error + + // Unsubscribe removes a subscription channel for an organization. + Unsubscribe(orgID OrganizationID, eventChan chan<- []Event) error + + // Cleanup removes old events based on retention policy. + // For SQLite: deletes events older than specified time + // For message brokers: may be a no-op if broker handles retention + Cleanup(ctx context.Context, olderThan time.Time) error + + // CleanupRange removes events within a specific time range. + CleanupRange(ctx context.Context, from, to time.Time) error + + // Close gracefully shuts down the backend. + Close() error +} + +// BackendType represents the type of message broker backend +type BackendType string + +const ( + // BackendTypeSQLite uses SQLite with polling for event delivery + BackendTypeSQLite BackendType = "sqlite" + // BackendTypeNATS uses NATS JetStream for event delivery + BackendTypeNATS BackendType = "nats" + // BackendTypeAzureServiceBus uses Azure Service Bus for event delivery + BackendTypeAzureServiceBus BackendType = "azure-servicebus" +) + +// BackendConfig holds common configuration for all backends +type BackendConfig struct { + // Type specifies which backend implementation to use + Type BackendType + + // SQLite-specific configuration + SQLite *SQLiteBackendConfig + + // NATS-specific configuration (for future use) + // NATS *NATSBackendConfig + + // Azure Service Bus configuration (for future use) + // AzureServiceBus *AzureServiceBusConfig +} + +// SQLiteBackendConfig holds SQLite-specific configuration +type SQLiteBackendConfig struct { + // PollInterval is how often to poll for state changes + PollInterval time.Duration + // RetentionPeriod is how long to keep events + RetentionPeriod time.Duration + // CleanupInterval is how often to run automatic cleanup + CleanupInterval time.Duration +} + +// DefaultSQLiteBackendConfig returns sensible defaults for SQLite backend +func DefaultSQLiteBackendConfig() *SQLiteBackendConfig { + return &SQLiteBackendConfig{ + PollInterval: time.Second * 5, + CleanupInterval: time.Minute * 10, + RetentionPeriod: time.Hour, + } +} diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index 62a625d27..3f2fc351f 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -3,187 +3,95 @@ package eventhub import ( "context" "database/sql" - "fmt" - "sync" "time" "go.uber.org/zap" ) // eventHub is the main implementation of EventHub interface +// It delegates to a Backend implementation for actual message broker operations type eventHub struct { - db *sql.DB - store *store - registry *organizationRegistry - poller *poller - config Config - logger *zap.Logger - - cleanupCtx context.Context - cleanupCancel context.CancelFunc - wg sync.WaitGroup - initialized bool - mu sync.RWMutex + backend EventhubImpl + logger *zap.Logger + + initialized bool } -// New creates a new EventHub instance +// New creates a new EventHub instance with SQLite backend (default) +// This maintains backward compatibility with existing code func New(db *sql.DB, logger *zap.Logger, config Config) EventHub { - registry := newOrganizationRegistry() - store := newStore(db, logger) + sqliteConfig := &SQLiteBackendConfig{ + PollInterval: config.PollInterval, + CleanupInterval: config.CleanupInterval, + RetentionPeriod: config.RetentionPeriod, + } + backend := NewSQLiteBackend(db, logger, sqliteConfig) + return &eventHub{ + backend: backend, + logger: logger, + } +} +// NewWithBackend creates a new EventHub instance with a custom backend +// Use this to provide alternative message broker implementations (NATS, Azure Service Bus, etc.) +func NewWithBackend(backend EventhubImpl, logger *zap.Logger) EventHub { return &eventHub{ - db: db, - store: store, - registry: registry, - config: config, - logger: logger, + backend: backend, + logger: logger, } } // Initialize sets up the EventHub and starts background workers func (eh *eventHub) Initialize(ctx context.Context) error { - eh.mu.Lock() - defer eh.mu.Unlock() - if eh.initialized { return nil } eh.logger.Info("Initializing EventHub") - // Create and start poller - eh.poller = newPoller(eh.store, eh.registry, eh.config, eh.logger) - eh.poller.start(ctx) - - // Start cleanup goroutine - eh.cleanupCtx, eh.cleanupCancel = context.WithCancel(ctx) - eh.wg.Add(1) - go eh.cleanupLoop() + if err := eh.backend.Initialize(ctx); err != nil { + return err + } eh.initialized = true - eh.logger.Info("EventHub initialized successfully", - zap.Duration("pollInterval", eh.config.PollInterval), - ) + eh.logger.Info("EventHub initialized successfully") return nil } // RegisterOrganization registers a new organization with the EventHub func (eh *eventHub) RegisterOrganization(organizationID OrganizationID) error { ctx := context.Background() - - // Register organization in registry - if err := eh.registry.register(organizationID); err != nil { - return err - } - - // Initialize empty state in database - if err := eh.store.initializeOrgState(ctx, organizationID); err != nil { - return fmt.Errorf("failed to initialize state: %w", err) - } - - eh.logger.Info("Organization registered", - zap.String("organization", string(organizationID)), - ) - - return nil + return eh.backend.RegisterOrganization(ctx, organizationID) } // PublishEvent publishes an event for an organization -// Note: Organization state and events are updated ATOMICALLY in a transaction func (eh *eventHub) PublishEvent(ctx context.Context, organizationID OrganizationID, eventType EventType, action, entityID string, eventData []byte) error { - - // Verify organization is registered - _, err := eh.registry.get(organizationID) - if err != nil { - return err - } - - // Publish atomically (event + state update in transaction) - version, err := eh.store.publishEventAtomic(ctx, organizationID, eventType, action, entityID, eventData) - if err != nil { - return fmt.Errorf("failed to publish event: %w", err) - } - - eh.logger.Debug("Event published", - zap.String("organization", string(organizationID)), - zap.String("eventType", string(eventType)), - zap.String("action", action), - zap.String("entityID", entityID), - zap.String("version", version), - ) - - return nil + return eh.backend.Publish(ctx, organizationID, eventType, action, entityID, eventData) } // Subscribe registers a channel to receive events for an organization -// Subscriber receives ALL event types and should filter by EventType if needed func (eh *eventHub) Subscribe(organizationID OrganizationID, eventChan chan<- []Event) error { - if err := eh.registry.addSubscriber(organizationID, eventChan); err != nil { - return err - } - - eh.logger.Info("Subscription registered", - zap.String("organization", string(organizationID)), - ) - - return nil + return eh.backend.Subscribe(organizationID, eventChan) } // CleanUpEvents removes events from the unified events table within the specified time range func (eh *eventHub) CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error { - _, err := eh.store.cleanupEvents(ctx, timeFrom, timeEnd) - if err != nil { - eh.logger.Error("Failed to cleanup events", zap.Error(err)) - return err - } - return nil -} - -// cleanupLoop runs periodic cleanup of old events -func (eh *eventHub) cleanupLoop() { - defer eh.wg.Done() - - ticker := time.NewTicker(eh.config.CleanupInterval) - defer ticker.Stop() - - for { - select { - case <-eh.cleanupCtx.Done(): - return - case <-ticker.C: - cutoff := time.Now().Add(-eh.config.RetentionPeriod) - if err := eh.store.cleanupAllOrganizations(eh.cleanupCtx, cutoff); err != nil { - eh.logger.Error("Periodic cleanup failed", zap.Error(err)) - } - } - } + return eh.backend.CleanupRange(ctx, timeFrom, timeEnd) } // Close gracefully shuts down the EventHub func (eh *eventHub) Close() error { - eh.mu.Lock() - defer eh.mu.Unlock() - if !eh.initialized { return nil } eh.logger.Info("Shutting down EventHub") - // Stop cleanup loop - if eh.cleanupCancel != nil { - eh.cleanupCancel() - } - - // Stop poller - if eh.poller != nil { - eh.poller.stop() + if err := eh.backend.Close(); err != nil { + return err } - // Wait for goroutines - eh.wg.Wait() - eh.initialized = false eh.logger.Info("EventHub shutdown complete") return nil diff --git a/gateway/gateway-controller/pkg/eventhub/poller.go b/gateway/gateway-controller/pkg/eventhub/poller.go deleted file mode 100644 index cbf678b9b..000000000 --- a/gateway/gateway-controller/pkg/eventhub/poller.go +++ /dev/null @@ -1,150 +0,0 @@ -package eventhub - -import ( - "context" - "sync" - "time" - - "go.uber.org/zap" -) - -// poller handles background polling for state changes and event delivery -type poller struct { - store *store - registry *organizationRegistry - config Config - logger *zap.Logger - - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup -} - -// newPoller creates a new event poller -func newPoller(store *store, registry *organizationRegistry, config Config, logger *zap.Logger) *poller { - return &poller{ - store: store, - registry: registry, - config: config, - logger: logger, - } -} - -// start begins the poller background worker -func (p *poller) start(ctx context.Context) { - p.ctx, p.cancel = context.WithCancel(ctx) - - p.wg.Add(1) - go p.pollLoop() - - p.logger.Info("Poller started", zap.Duration("interval", p.config.PollInterval)) -} - -// pollLoop runs the main polling loop -func (p *poller) pollLoop() { - defer p.wg.Done() - - ticker := time.NewTicker(p.config.PollInterval) - defer ticker.Stop() - - for { - select { - case <-p.ctx.Done(): - return - case <-ticker.C: - p.pollAllOrganizations() - } - } -} - -// pollAllOrganizations checks all organizations for state changes using single query -func (p *poller) pollAllOrganizations() { - ctx := p.ctx - - // STEP 1: Single query for ALL organization states - states, err := p.store.getAllStates(ctx) - if err != nil { - p.logger.Error("Failed to fetch all states", zap.Error(err)) - return - } - - // STEP 2: Loop through each organization sequentially - for _, state := range states { - orgID := OrganizationID(state.Organization) - - // Get the organization from registry - org, err := p.registry.get(orgID) - if err != nil { - // Organization not registered with subscribers, skip - continue - } - - // Check if version changed - if state.VersionID == org.knownVersion { - // No changes - continue - } - - p.logger.Debug("State change detected", - zap.String("organization", string(orgID)), - zap.String("oldVersion", org.knownVersion), - zap.String("newVersion", state.VersionID), - ) - - // Fetch events since last poll - events, err := p.store.getEventsSince(ctx, orgID, org.lastPolled) - if err != nil { - p.logger.Error("Failed to fetch events", - zap.String("organization", string(orgID)), - zap.Error(err)) - continue - } - - if len(events) > 0 { - // Deliver events to subscribers - p.deliverEvents(org, events) - } - - // Update poll state - org.updatePollState(state.VersionID, time.Now()) - } -} - -// deliverEvents sends events to all subscribers of an organization -func (p *poller) deliverEvents(org *organization, events []Event) { - subscribers := org.getSubscribers() - - if len(subscribers) == 0 { - p.logger.Debug("No subscribers for organization", - zap.String("organization", string(org.id)), - zap.Int("events", len(events)), - ) - return - } - - // Deliver ALL events (all event types) to subscribers - // Consumers are responsible for filtering by EventType if needed - for _, ch := range subscribers { - select { - case ch <- events: - p.logger.Debug("Delivered events to subscriber", - zap.String("organization", string(org.id)), - zap.Int("events", len(events)), - ) - default: - p.logger.Warn("Subscriber channel full, dropping events", - zap.String("organization", string(org.id)), - zap.Int("events", len(events)), - ) - } - } -} - -// stop gracefully stops the poller -func (p *poller) stop() { - if p.cancel != nil { - p.cancel() - } - p.wg.Wait() - p.logger.Info("Poller stopped") -} diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go new file mode 100644 index 000000000..186c882be --- /dev/null +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -0,0 +1,417 @@ +package eventhub + +import ( + "context" + "database/sql" + "fmt" + "sync" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" +) + +// SQLiteBackend implements the Backend interface using SQLite with polling +type SQLiteBackend struct { + db *sql.DB + config *SQLiteBackendConfig + logger *zap.Logger + registry *organizationRegistry + + // Polling state + pollerCtx context.Context + pollerCancel context.CancelFunc + cleanupCtx context.Context + cleanupCancel context.CancelFunc + wg sync.WaitGroup + + initialized bool + mu sync.RWMutex +} + +// NewSQLiteBackend creates a new SQLite-based backend +func NewSQLiteBackend(db *sql.DB, logger *zap.Logger, config *SQLiteBackendConfig) *SQLiteBackend { + if config == nil { + config = DefaultSQLiteBackendConfig() + } + return &SQLiteBackend{ + db: db, + config: config, + logger: logger, + registry: newOrganizationRegistry(), + } +} + +// Initialize sets up the SQLite backend and starts background workers +func (b *SQLiteBackend) Initialize(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + + if b.initialized { + return nil + } + + b.logger.Info("Initializing SQLite backend") + + // Start poller + b.pollerCtx, b.pollerCancel = context.WithCancel(ctx) + b.wg.Add(1) + go b.pollLoop() + + // Start cleanup goroutine + b.cleanupCtx, b.cleanupCancel = context.WithCancel(ctx) + b.wg.Add(1) + go b.cleanupLoop() + + b.initialized = true + b.logger.Info("SQLite backend initialized", + zap.Duration("pollInterval", b.config.PollInterval), + zap.Duration("cleanupInterval", b.config.CleanupInterval), + zap.Duration("retentionPeriod", b.config.RetentionPeriod), + ) + return nil +} + +// RegisterOrganization creates the necessary resources for an organization +func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID OrganizationID) error { + // Register in local registry + if err := b.registry.register(orgID); err != nil { + return err + } + + // Initialize state in database + query := ` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO NOTHING + ` + _, err := b.db.ExecContext(ctx, query, string(orgID), "", time.Now()) + if err != nil { + return fmt.Errorf("failed to initialize organization state: %w", err) + } + + b.logger.Info("Organization registered in SQLite backend", + zap.String("organization", string(orgID)), + ) + return nil +} + +// Publish publishes an event for an organization +func (b *SQLiteBackend) Publish(ctx context.Context, orgID OrganizationID, + eventType EventType, action, entityID string, eventData []byte) error { + + // Verify organization is registered + _, err := b.registry.get(orgID) + if err != nil { + return err + } + + // Publish atomically (event + state update in transaction) + tx, err := b.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + now := time.Now() + + // Insert event + insertQuery := ` + INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, + event_type, action, entity_id, event_data) + VALUES (?, ?, ?, ?, ?, ?, ?) + ` + _, err = tx.ExecContext(ctx, insertQuery, + string(orgID), now, now, string(eventType), action, entityID, eventData) + if err != nil { + return fmt.Errorf("failed to record event: %w", err) + } + + // Update organization state version + newVersion := uuid.New().String() + updateQuery := ` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at + ` + _, err = tx.ExecContext(ctx, updateQuery, string(orgID), newVersion, now) + if err != nil { + return fmt.Errorf("failed to update state: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + b.logger.Debug("Event published", + zap.String("organization", string(orgID)), + zap.String("eventType", string(eventType)), + zap.String("action", action), + zap.String("entityID", entityID), + zap.String("version", newVersion), + ) + + return nil +} + +// Subscribe registers a channel to receive events for an organization +func (b *SQLiteBackend) Subscribe(orgID OrganizationID, eventChan chan<- []Event) error { + if err := b.registry.addSubscriber(orgID, eventChan); err != nil { + return err + } + + b.logger.Info("Subscription registered", + zap.String("organization", string(orgID)), + ) + return nil +} + +// Unsubscribe removes a subscription channel for an organization +func (b *SQLiteBackend) Unsubscribe(orgID OrganizationID, eventChan chan<- []Event) error { + if err := b.registry.removeSubscriber(orgID, eventChan); err != nil { + return err + } + + b.logger.Info("Subscription removed", + zap.String("organization", string(orgID)), + ) + return nil +} + +// Cleanup removes old events based on retention policy +func (b *SQLiteBackend) Cleanup(ctx context.Context, olderThan time.Time) error { + query := `DELETE FROM events WHERE processed_timestamp < ?` + result, err := b.db.ExecContext(ctx, query, olderThan) + if err != nil { + return fmt.Errorf("failed to cleanup old events: %w", err) + } + + deleted, _ := result.RowsAffected() + b.logger.Info("Cleaned up old events", + zap.Int64("deleted", deleted), + zap.Time("olderThan", olderThan), + ) + return nil +} + +// CleanupRange removes events within a specific time range +func (b *SQLiteBackend) CleanupRange(ctx context.Context, from, to time.Time) error { + query := `DELETE FROM events WHERE processed_timestamp >= ? AND processed_timestamp <= ?` + result, err := b.db.ExecContext(ctx, query, from, to) + if err != nil { + return fmt.Errorf("failed to cleanup events: %w", err) + } + + deleted, _ := result.RowsAffected() + b.logger.Info("Cleaned up events in range", + zap.Int64("deleted", deleted), + zap.Time("from", from), + zap.Time("to", to), + ) + return nil +} + +// Close gracefully shuts down the SQLite backend +func (b *SQLiteBackend) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + + if !b.initialized { + return nil + } + + b.logger.Info("Shutting down SQLite backend") + + // Stop poller + if b.pollerCancel != nil { + b.pollerCancel() + } + + // Stop cleanup loop + if b.cleanupCancel != nil { + b.cleanupCancel() + } + + // Wait for goroutines + b.wg.Wait() + + b.initialized = false + b.logger.Info("SQLite backend shutdown complete") + return nil +} + +// pollLoop runs the main polling loop for state changes +func (b *SQLiteBackend) pollLoop() { + defer b.wg.Done() + + ticker := time.NewTicker(b.config.PollInterval) + defer ticker.Stop() + + b.logger.Info("SQLite poller started", zap.Duration("interval", b.config.PollInterval)) + + for { + select { + case <-b.pollerCtx.Done(): + b.logger.Info("SQLite poller stopped") + return + case <-ticker.C: + b.pollAllOrganizations() + } + } +} + +// pollAllOrganizations checks all organizations for state changes +func (b *SQLiteBackend) pollAllOrganizations() { + ctx := b.pollerCtx + + // Single query for ALL organization states + states, err := b.getAllStates(ctx) + if err != nil { + b.logger.Error("Failed to fetch all states", zap.Error(err)) + return + } + + // Check each organization for changes + for _, state := range states { + orgID := OrganizationID(state.Organization) + + org, err := b.registry.get(orgID) + if err != nil { + // Organization not registered with subscribers, skip + continue + } + + // Check if version changed + if state.VersionID == org.knownVersion { + continue + } + + b.logger.Debug("State change detected", + zap.String("organization", string(orgID)), + zap.String("oldVersion", org.knownVersion), + zap.String("newVersion", state.VersionID), + ) + + // Fetch events since last poll + events, err := b.getEventsSince(ctx, orgID, org.lastPolled) + if err != nil { + b.logger.Error("Failed to fetch events", + zap.String("organization", string(orgID)), + zap.Error(err)) + continue + } + + if len(events) > 0 { + b.deliverEvents(org, events) + } + + org.updatePollState(state.VersionID, time.Now()) + } +} + +// getAllStates retrieves all organization states +func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, error) { + query := ` + SELECT organization, version_id, updated_at + FROM organization_states + ORDER BY organization + ` + + rows, err := b.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query all states: %w", err) + } + defer rows.Close() + + var states []OrganizationState + for rows.Next() { + var state OrganizationState + if err := rows.Scan(&state.Organization, &state.VersionID, &state.UpdatedAt); err != nil { + return nil, fmt.Errorf("failed to scan state: %w", err) + } + states = append(states, state) + } + return states, rows.Err() +} + +// getEventsSince retrieves events for an organization after a given timestamp +func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID OrganizationID, since time.Time) ([]Event, error) { + query := ` + SELECT processed_timestamp, originated_timestamp, event_type, + action, entity_id, event_data + FROM events + WHERE organization_id = ? AND processed_timestamp > ? + ORDER BY processed_timestamp ASC + ` + + rows, err := b.db.QueryContext(ctx, query, string(orgID), since) + if err != nil { + return nil, fmt.Errorf("failed to query events: %w", err) + } + defer rows.Close() + + var events []Event + for rows.Next() { + var e Event + var eventTypeStr string + e.OrganizationID = orgID + + if err := rows.Scan(&e.ProcessedTimestamp, &e.OriginatedTimestamp, + &eventTypeStr, &e.Action, &e.EntityID, &e.EventData); err != nil { + return nil, fmt.Errorf("failed to scan event: %w", err) + } + e.EventType = EventType(eventTypeStr) + events = append(events, e) + } + return events, rows.Err() +} + +// deliverEvents sends events to all subscribers of an organization +func (b *SQLiteBackend) deliverEvents(org *organization, events []Event) { + subscribers := org.getSubscribers() + + if len(subscribers) == 0 { + b.logger.Debug("No subscribers for organization", + zap.String("organization", string(org.id)), + zap.Int("events", len(events)), + ) + return + } + + for _, ch := range subscribers { + select { + case ch <- events: + b.logger.Debug("Delivered events to subscriber", + zap.String("organization", string(org.id)), + zap.Int("events", len(events)), + ) + default: + b.logger.Warn("Subscriber channel full, dropping events", + zap.String("organization", string(org.id)), + zap.Int("events", len(events)), + ) + } + } +} + +// cleanupLoop runs periodic cleanup of old events +func (b *SQLiteBackend) cleanupLoop() { + defer b.wg.Done() + + ticker := time.NewTicker(b.config.CleanupInterval) + defer ticker.Stop() + + for { + select { + case <-b.cleanupCtx.Done(): + return + case <-ticker.C: + cutoff := time.Now().Add(-b.config.RetentionPeriod) + if err := b.Cleanup(b.cleanupCtx, cutoff); err != nil { + b.logger.Error("Periodic cleanup failed", zap.Error(err)) + } + } + } +} diff --git a/gateway/gateway-controller/pkg/eventhub/store.go b/gateway/gateway-controller/pkg/eventhub/store.go deleted file mode 100644 index f9357e868..000000000 --- a/gateway/gateway-controller/pkg/eventhub/store.go +++ /dev/null @@ -1,244 +0,0 @@ -package eventhub - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/google/uuid" - "go.uber.org/zap" -) - -// store handles database operations for EventHub -type store struct { - db *sql.DB - logger *zap.Logger -} - -// newStore creates a new database store -func newStore(db *sql.DB, logger *zap.Logger) *store { - return &store{ - db: db, - logger: logger, - } -} - -// initializeOrgState creates an empty state entry for an organization -func (s *store) initializeOrgState(ctx context.Context, orgID OrganizationID) error { - query := ` - INSERT INTO organization_states (organization, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(organization) - DO NOTHING - ` - - _, err := s.db.ExecContext(ctx, query, string(orgID), "", time.Now()) - if err != nil { - return fmt.Errorf("failed to initialize organization state: %w", err) - } - return nil -} - -// getAllStates retrieves all organization states in a single query -func (s *store) getAllStates(ctx context.Context) ([]OrganizationState, error) { - query := ` - SELECT organization, version_id, updated_at - FROM organization_states - ORDER BY organization - ` - - rows, err := s.db.QueryContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to query all states: %w", err) - } - defer rows.Close() - - var states []OrganizationState - for rows.Next() { - var state OrganizationState - if err := rows.Scan(&state.Organization, &state.VersionID, &state.UpdatedAt); err != nil { - return nil, fmt.Errorf("failed to scan state: %w", err) - } - states = append(states, state) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating states: %w", err) - } - - return states, nil -} - -// getState retrieves the current state for an organization -func (s *store) getState(ctx context.Context, orgID OrganizationID) (*OrganizationState, error) { - query := ` - SELECT organization, version_id, updated_at - FROM organization_states - WHERE organization = ? - ` - - var state OrganizationState - err := s.db.QueryRowContext(ctx, query, string(orgID)).Scan( - &state.Organization, &state.VersionID, &state.UpdatedAt, - ) - if err == sql.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("failed to get state: %w", err) - } - - return &state, nil -} - -// publishEventAtomic records an event and updates state in a single transaction -func (s *store) publishEventAtomic(ctx context.Context, orgID OrganizationID, eventType EventType, - action, entityID string, eventData []byte) (string, error) { - - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return "", fmt.Errorf("failed to begin transaction: %w", err) - } - defer tx.Rollback() - - now := time.Now() - - // Step 1: Insert event into unified events table - insertQuery := ` - INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, - event_type, action, entity_id, event_data) - VALUES (?, ?, ?, ?, ?, ?, ?) - ` - - _, err = tx.ExecContext(ctx, insertQuery, - string(orgID), - now, - now, - string(eventType), - action, - entityID, - eventData, - ) - if err != nil { - return "", fmt.Errorf("failed to record event: %w", err) - } - - // Step 2: Update organization state version - newVersion := uuid.New().String() - - updateQuery := ` - INSERT INTO organization_states (organization, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(organization) - DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at - ` - - _, err = tx.ExecContext(ctx, updateQuery, string(orgID), newVersion, now) - if err != nil { - return "", fmt.Errorf("failed to update state: %w", err) - } - - // Commit transaction - if err := tx.Commit(); err != nil { - return "", fmt.Errorf("failed to commit transaction: %w", err) - } - - s.logger.Debug("Published event atomically", - zap.String("organization", string(orgID)), - zap.String("eventType", string(eventType)), - zap.String("action", action), - zap.String("entityID", entityID), - zap.String("version", newVersion), - ) - - return newVersion, nil -} - -// getEventsSince retrieves events for an organization after a given timestamp -func (s *store) getEventsSince(ctx context.Context, orgID OrganizationID, since time.Time) ([]Event, error) { - query := ` - SELECT processed_timestamp, originated_timestamp, event_type, - action, entity_id, event_data - FROM events - WHERE organization_id = ? AND processed_timestamp > ? - ORDER BY processed_timestamp ASC - ` - - rows, err := s.db.QueryContext(ctx, query, string(orgID), since) - if err != nil { - return nil, fmt.Errorf("failed to query events: %w", err) - } - defer rows.Close() - - var events []Event - for rows.Next() { - var e Event - var eventTypeStr string - e.OrganizationID = orgID - - if err := rows.Scan(&e.ProcessedTimestamp, &e.OriginatedTimestamp, - &eventTypeStr, &e.Action, &e.EntityID, &e.EventData); err != nil { - return nil, fmt.Errorf("failed to scan event: %w", err) - } - e.EventType = EventType(eventTypeStr) - events = append(events, e) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating events: %w", err) - } - - return events, nil -} - -// cleanupEvents removes events from the unified table within the specified time range -func (s *store) cleanupEvents(ctx context.Context, timeFrom, timeEnd time.Time) (int64, error) { - query := ` - DELETE FROM events - WHERE processed_timestamp >= ? AND processed_timestamp <= ? - ` - - result, err := s.db.ExecContext(ctx, query, timeFrom, timeEnd) - if err != nil { - return 0, fmt.Errorf("failed to cleanup events: %w", err) - } - - deleted, err := result.RowsAffected() - if err != nil { - return 0, fmt.Errorf("failed to get deleted count: %w", err) - } - - s.logger.Info("Cleaned up events", - zap.Int64("deleted", deleted), - zap.Time("from", timeFrom), - zap.Time("to", timeEnd), - ) - - return deleted, nil -} - -// cleanupAllOrganizations removes old events from the unified events table -func (s *store) cleanupAllOrganizations(ctx context.Context, olderThan time.Time) error { - query := ` - DELETE FROM events - WHERE processed_timestamp < ? - ` - - result, err := s.db.ExecContext(ctx, query, olderThan) - if err != nil { - return fmt.Errorf("failed to cleanup old events: %w", err) - } - - deleted, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get deleted count: %w", err) - } - - s.logger.Info("Cleaned up old events across all organizations", - zap.Int64("deleted", deleted), - zap.Time("olderThan", olderThan), - ) - - return nil -} diff --git a/gateway/gateway-controller/pkg/eventhub/topic.go b/gateway/gateway-controller/pkg/eventhub/topic.go index 7af18ef0a..824b66f25 100644 --- a/gateway/gateway-controller/pkg/eventhub/topic.go +++ b/gateway/gateway-controller/pkg/eventhub/topic.go @@ -81,6 +81,29 @@ func (r *organizationRegistry) addSubscriber(id OrganizationID, ch chan<- []Even return nil } +// removeSubscriber removes a subscription channel from an organization +func (r *organizationRegistry) removeSubscriber(id OrganizationID, ch chan<- []Event) error { + r.mu.RLock() + org, exists := r.orgs[id] + r.mu.RUnlock() + + if !exists { + return ErrOrganizationNotFound + } + + org.subscriberMu.Lock() + defer org.subscriberMu.Unlock() + + // Find and remove the subscriber + for i, sub := range org.subscribers { + if sub == ch { + org.subscribers = append(org.subscribers[:i], org.subscribers[i+1:]...) + return nil + } + } + return nil // Not found is not an error +} + // getAll returns all registered organizations func (r *organizationRegistry) getAll() []*organization { r.mu.RLock() From 52ec10fc99d8a4c59f6912a38c20b011c485896a Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 4 Jan 2026 10:15:53 +0530 Subject: [PATCH 09/24] Fix API Deployment via eventhub --- .../gateway-controller/cmd/controller/main.go | 19 +++- .../pkg/api/handlers/handlers.go | 5 +- .../gateway-controller/pkg/config/config.go | 14 +-- .../pkg/controlplane/client.go | 3 +- .../pkg/storage/interface.go | 5 ++ .../gateway-controller/pkg/storage/sqlite.go | 6 ++ .../pkg/utils/api_deployment.go | 90 ++++++++++++++----- .../utils/websub_topic_registration_test.go | 8 +- 8 files changed, 113 insertions(+), 37 deletions(-) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 2ac60f471..0b61bc32d 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -20,6 +20,7 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/middleware" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/controlplane" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/logger" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" @@ -107,6 +108,22 @@ func main() { // Initialize in-memory API key store for xDS apiKeyStore := storage.NewAPIKeyStore(log) + // Initialize EventHub if multi-tenant mode is enabled + var eventHub eventhub.EventHub + if cfg.GatewayController.Server.EnableMultiTenantMode { + if cfg.IsPersistentMode() && db != nil { + log.Info("Initializing EventHub for multi-tenant mode") + eventHub = eventhub.New(db.GetDB(), log, eventhub.DefaultConfig()) + ctx := context.Background() + if err := eventHub.Initialize(ctx); err != nil { + log.Fatal("Failed to initialize EventHub", zap.Error(err)) + } + log.Info("EventHub initialized successfully") + } else { + log.Fatal("EventHub requires persistent storage. Multi-tenant mode will not function correctly in memory-only mode.") + } + } + // Load configurations from database on startup (if persistent mode) if cfg.IsPersistentMode() && db != nil { log.Info("Loading configurations from database") @@ -301,7 +318,7 @@ func main() { // Initialize API server with the configured validator and API key manager apiServer := handlers.NewAPIServer(configStore, db, snapshotManager, policyManager, log, cpClient, - policyDefinitions, templateDefinitions, validator, &cfg.GatewayController.Router, apiKeyXDSManager) + policyDefinitions, templateDefinitions, validator, &cfg.GatewayController.Router, apiKeyXDSManager, eventHub, cfg.GatewayController.Server.EnableMultiTenantMode) // Register API routes (includes certificate management endpoints from OpenAPI spec) api.RegisterHandlers(router, apiServer) diff --git a/gateway/gateway-controller/pkg/api/handlers/handlers.go b/gateway/gateway-controller/pkg/api/handlers/handlers.go index b25d42326..3e1c5b641 100644 --- a/gateway/gateway-controller/pkg/api/handlers/handlers.go +++ b/gateway/gateway-controller/pkg/api/handlers/handlers.go @@ -39,6 +39,7 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/middleware" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/controlplane" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" @@ -82,8 +83,10 @@ func NewAPIServer( validator config.Validator, routerConfig *config.RouterConfig, apiKeyXDSManager *apikeyxds.APIKeyStateManager, + eventHub eventhub.EventHub, + enableMultiTenantMode bool, ) *APIServer { - deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig) + deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig, eventHub, enableMultiTenantMode) server := &APIServer{ store: store, db: db, diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 90fe7a1f8..c952f7aaf 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -117,9 +117,10 @@ type TracingConfig struct { // ServerConfig holds server-related configuration type ServerConfig struct { - APIPort int `koanf:"api_port"` - XDSPort int `koanf:"xds_port"` - ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` + APIPort int `koanf:"api_port"` + XDSPort int `koanf:"xds_port"` + ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` + EnableMultiTenantMode bool `koanf:"enable_multi_tenant_mode"` } // PolicyServerConfig holds policy xDS server-related configuration @@ -359,9 +360,10 @@ func defaultConfig() *Config { return &Config{ GatewayController: GatewayController{ Server: ServerConfig{ - APIPort: 9090, - XDSPort: 18000, - ShutdownTimeout: 15 * time.Second, + APIPort: 9090, + XDSPort: 18000, + ShutdownTimeout: 15 * time.Second, + EnableMultiTenantMode: false, }, PolicyServer: PolicyServerConfig{ Enabled: true, diff --git a/gateway/gateway-controller/pkg/controlplane/client.go b/gateway/gateway-controller/pkg/controlplane/client.go index ad82be485..6fd6d4d5b 100644 --- a/gateway/gateway-controller/pkg/controlplane/client.go +++ b/gateway/gateway-controller/pkg/controlplane/client.go @@ -126,7 +126,8 @@ func NewClient( snapshotManager: snapshotManager, parser: config.NewParser(), validator: validator, - deploymentService: utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig), + // TODO: (VirajSalaka) Decide on behavior when controlplane is involved. + deploymentService: utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig, nil, false), state: &ConnectionState{ Current: Disconnected, Conn: nil, diff --git a/gateway/gateway-controller/pkg/storage/interface.go b/gateway/gateway-controller/pkg/storage/interface.go index fcbc80ef1..844491f48 100644 --- a/gateway/gateway-controller/pkg/storage/interface.go +++ b/gateway/gateway-controller/pkg/storage/interface.go @@ -19,6 +19,8 @@ package storage import ( + "database/sql" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" ) @@ -227,4 +229,7 @@ type Storage interface { // Should be called during graceful shutdown. // Implementations should ensure all pending writes are flushed. Close() error + + // GetDB exposes the underlying *sql.DB for advanced operations. + GetDB() *sql.DB } diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index 30a06d4bb..d732c4343 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -1761,3 +1761,9 @@ func LoadAPIKeysFromDatabase(storage Storage, configStore *ConfigStore, apiKeySt return nil } + +// GetDB returns the underlying *sql.DB instance +// This is used for eventhub initialization +func (s *SQLiteStorage) GetDB() *sql.DB { + return s.db +} diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 46af53f27..a67a54741 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -31,6 +31,7 @@ import ( "github.com/google/uuid" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" @@ -54,13 +55,15 @@ type APIDeploymentResult struct { // APIDeploymentService provides utilities for API configuration deployment type APIDeploymentService struct { - store *storage.ConfigStore - db storage.Storage - snapshotManager *xds.SnapshotManager - parser *config.Parser - validator config.Validator - routerConfig *config.RouterConfig - httpClient *http.Client + store *storage.ConfigStore + db storage.Storage + snapshotManager *xds.SnapshotManager + parser *config.Parser + validator config.Validator + routerConfig *config.RouterConfig + httpClient *http.Client + eventHub eventhub.EventHub + enableMultiTenantMode bool } // NewAPIDeploymentService creates a new API deployment service @@ -70,15 +73,19 @@ func NewAPIDeploymentService( snapshotManager *xds.SnapshotManager, validator config.Validator, routerConfig *config.RouterConfig, + eventHub eventhub.EventHub, + enableMultiTenantMode bool, ) *APIDeploymentService { return &APIDeploymentService{ - store: store, - db: db, - snapshotManager: snapshotManager, - parser: config.NewParser(), - validator: validator, - httpClient: &http.Client{Timeout: 10 * time.Second}, - routerConfig: routerConfig, + store: store, + db: db, + snapshotManager: snapshotManager, + parser: config.NewParser(), + validator: validator, + httpClient: &http.Client{Timeout: 10 * time.Second}, + routerConfig: routerConfig, + eventHub: eventHub, + enableMultiTenantMode: enableMultiTenantMode, } } @@ -268,18 +275,53 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams zap.String("correlation_id", params.CorrelationID)) } - // Update xDS snapshot asynchronously - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + // Update xDS snapshot or publish event based on multi-tenant mode + if !s.enableMultiTenantMode { + // Single-tenant mode: Update xDS snapshot asynchronously + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := s.snapshotManager.UpdateSnapshot(ctx, params.CorrelationID); err != nil { + params.Logger.Error("Failed to update xDS snapshot", + zap.Error(err), + zap.String("api_id", apiID), + zap.String("correlation_id", params.CorrelationID)) + } + }() + } else { + // Multi-tenant mode: Publish event to eventhub + if s.eventHub != nil { + // Determine action based on whether it's an update or create + action := "CREATE" + if isUpdate { + action = "UPDATE" + } - if err := s.snapshotManager.UpdateSnapshot(ctx, params.CorrelationID); err != nil { - params.Logger.Error("Failed to update xDS snapshot", - zap.Error(err), - zap.String("api_id", apiID), - zap.String("correlation_id", params.CorrelationID)) + // Use default organization ID (can be made configurable in future) + organizationID := eventhub.OrganizationID("default") + + // Publish event with empty payload as per requirements + ctx := context.Background() + if err := s.eventHub.PublishEvent(ctx, organizationID, eventhub.EventTypeAPI, action, apiID, []byte{}); err != nil { + params.Logger.Error("Failed to publish event to eventhub", + zap.Error(err), + zap.String("api_id", apiID), + zap.String("action", action), + zap.String("organization_id", string(organizationID)), + zap.String("correlation_id", params.CorrelationID)) + } else { + params.Logger.Info("Event published to eventhub", + zap.String("api_id", apiID), + zap.String("action", action), + zap.String("organization_id", string(organizationID)), + zap.String("correlation_id", params.CorrelationID)) + } + } else { + params.Logger.Warn("Multi-tenant mode enabled but eventhub is not initialized", + zap.String("api_id", apiID)) } - }() + } return &APIDeploymentResult{ StoredConfig: storedCfg, diff --git a/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go b/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go index 2297d9fa1..ce7122a61 100644 --- a/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go +++ b/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go @@ -18,7 +18,7 @@ func TestDeployAPIConfigurationWebSubKindTopicRegistration(t *testing.T) { db := &storage.SQLiteStorage{} snapshotManager := &xds.SnapshotManager{} validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, db, snapshotManager, validator, nil) + service := NewAPIDeploymentService(configStore, db, snapshotManager, validator, nil, nil, false) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub @@ -63,7 +63,7 @@ spec: func TestDeployAPIConfigurationWebSubKindRevisionDeployment(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, validator, nil, nil, false) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub @@ -145,7 +145,7 @@ spec: func TestTopicRegistrationForConcurrentAPIConfigs(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, validator, nil, nil, false) // Two different API YAMLs yamlA := `kind: async/websub @@ -249,7 +249,7 @@ spec: func TestTopicDeregistrationOnConfigDeletion(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, validator, nil, nil, false) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub From f498c26925f9766be69fd551bc1ba6eb7595a5cb Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 4 Jan 2026 10:40:36 +0530 Subject: [PATCH 10/24] Add todo comments --- gateway/gateway-controller/pkg/api/handlers/handlers.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gateway/gateway-controller/pkg/api/handlers/handlers.go b/gateway/gateway-controller/pkg/api/handlers/handlers.go index 3e1c5b641..309323167 100644 --- a/gateway/gateway-controller/pkg/api/handlers/handlers.go +++ b/gateway/gateway-controller/pkg/api/handlers/handlers.go @@ -516,6 +516,7 @@ func (s *APIServer) UpdateAPI(c *gin.Context, id string) { return } + // TODO: (VirajSalaka) Needs to implement this based on deploymentService DeployAPIConfiguration // Validate that the handle in the YAML matches the path parameter if apiConfig.Metadata.Name != "" { if apiConfig.Metadata.Name != handle { @@ -702,6 +703,7 @@ func (s *APIServer) UpdateAPI(c *gin.Context, id string) { correlationID := middleware.GetCorrelationID(c) // Update xDS snapshot asynchronously + // TODO: (VirajSalaka) Fix to work with eventhub go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -901,6 +903,7 @@ func (s *APIServer) DeleteAPI(c *gin.Context, id string) { correlationID := middleware.GetCorrelationID(c) // Update xDS snapshot asynchronously + // TODO: (VirajSalaka) Fix to work with eventhub go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -921,6 +924,7 @@ func (s *APIServer) DeleteAPI(c *gin.Context, id string) { }) // Remove derived policy configuration + // TODO: (VirajSalaka) Fix to work with eventhub if s.policyManager != nil { policyID := cfg.ID + "-policies" if err := s.policyManager.RemovePolicy(policyID); err != nil { From 90fb3fb9f27df5af1385547e98b973084eb0b9bb Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 4 Jan 2026 10:41:35 +0530 Subject: [PATCH 11/24] Update Inmemory Database via Eventhub all the time --- .../pkg/utils/api_deployment.go | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index a67a54741..57d21b7d2 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -393,22 +393,24 @@ func (s *APIDeploymentService) saveOrUpdateConfig(storedCfg *models.StoredConfig } // Try to add to in-memory store - if err := s.store.Add(storedCfg); err != nil { - // Check if it's a conflict (API already exists) - if storage.IsConflictError(err) { - logger.Info("API configuration already exists in memory, updating instead", - zap.String("api_id", storedCfg.ID), - zap.String("displayName", storedCfg.GetDisplayName()), - zap.String("version", storedCfg.GetVersion())) - - // Try to update instead - return s.updateExistingConfig(storedCfg, logger) - } else { - // Rollback database write (only if persistent mode) - if s.db != nil { - _ = s.db.DeleteConfig(storedCfg.ID) + if !s.enableMultiTenantMode { + if err := s.store.Add(storedCfg); err != nil { + // Check if it's a conflict (API already exists) + if storage.IsConflictError(err) { + logger.Info("API configuration already exists in memory, updating instead", + zap.String("api_id", storedCfg.ID), + zap.String("displayName", storedCfg.GetDisplayName()), + zap.String("version", storedCfg.GetVersion())) + + // Try to update instead + return s.updateExistingConfig(storedCfg, logger) + } else { + // Rollback database write (only if persistent mode) + if s.db != nil { + _ = s.db.DeleteConfig(storedCfg.ID) + } + return false, fmt.Errorf("failed to add config to memory store: %w", err) } - return false, fmt.Errorf("failed to add config to memory store: %w", err) } } From 0e5ed4f6d1673ed93b7b7c187d9eb5a5ffe7aaba Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Sun, 4 Jan 2026 11:22:29 +0530 Subject: [PATCH 12/24] Refactor - rename --- .../gateway-controller/cmd/controller/main.go | 7 ++-- .../gateway-controller/pkg/config/config.go | 16 +++---- .../pkg/utils/api_deployment.go | 42 +++++++++---------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 0b61bc32d..5e956a0e2 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/apikeyxds" "net/http" "os" "os/signal" @@ -12,6 +11,8 @@ import ( "syscall" "time" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/apikeyxds" + "github.com/gin-gonic/gin" "github.com/wso2/api-platform/common/authenticators" commonmodels "github.com/wso2/api-platform/common/models" @@ -110,7 +111,7 @@ func main() { // Initialize EventHub if multi-tenant mode is enabled var eventHub eventhub.EventHub - if cfg.GatewayController.Server.EnableMultiTenantMode { + if cfg.GatewayController.Server.EnableReplicaSync { if cfg.IsPersistentMode() && db != nil { log.Info("Initializing EventHub for multi-tenant mode") eventHub = eventhub.New(db.GetDB(), log, eventhub.DefaultConfig()) @@ -318,7 +319,7 @@ func main() { // Initialize API server with the configured validator and API key manager apiServer := handlers.NewAPIServer(configStore, db, snapshotManager, policyManager, log, cpClient, - policyDefinitions, templateDefinitions, validator, &cfg.GatewayController.Router, apiKeyXDSManager, eventHub, cfg.GatewayController.Server.EnableMultiTenantMode) + policyDefinitions, templateDefinitions, validator, &cfg.GatewayController.Router, apiKeyXDSManager, eventHub, cfg.GatewayController.Server.EnableReplicaSync) // Register API routes (includes certificate management endpoints from OpenAPI spec) api.RegisterHandlers(router, apiServer) diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index c952f7aaf..766281b8d 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -117,10 +117,10 @@ type TracingConfig struct { // ServerConfig holds server-related configuration type ServerConfig struct { - APIPort int `koanf:"api_port"` - XDSPort int `koanf:"xds_port"` - ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` - EnableMultiTenantMode bool `koanf:"enable_multi_tenant_mode"` + APIPort int `koanf:"api_port"` + XDSPort int `koanf:"xds_port"` + ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` + EnableReplicaSync bool `koanf:"enable_replica_sync"` } // PolicyServerConfig holds policy xDS server-related configuration @@ -360,10 +360,10 @@ func defaultConfig() *Config { return &Config{ GatewayController: GatewayController{ Server: ServerConfig{ - APIPort: 9090, - XDSPort: 18000, - ShutdownTimeout: 15 * time.Second, - EnableMultiTenantMode: false, + APIPort: 9090, + XDSPort: 18000, + ShutdownTimeout: 15 * time.Second, + EnableReplicaSync: false, }, PolicyServer: PolicyServerConfig{ Enabled: true, diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 57d21b7d2..ffe2ea1f3 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -55,15 +55,15 @@ type APIDeploymentResult struct { // APIDeploymentService provides utilities for API configuration deployment type APIDeploymentService struct { - store *storage.ConfigStore - db storage.Storage - snapshotManager *xds.SnapshotManager - parser *config.Parser - validator config.Validator - routerConfig *config.RouterConfig - httpClient *http.Client - eventHub eventhub.EventHub - enableMultiTenantMode bool + store *storage.ConfigStore + db storage.Storage + snapshotManager *xds.SnapshotManager + parser *config.Parser + validator config.Validator + routerConfig *config.RouterConfig + httpClient *http.Client + eventHub eventhub.EventHub + enableReplicaSync bool } // NewAPIDeploymentService creates a new API deployment service @@ -74,18 +74,18 @@ func NewAPIDeploymentService( validator config.Validator, routerConfig *config.RouterConfig, eventHub eventhub.EventHub, - enableMultiTenantMode bool, + enableReplicaSync bool, ) *APIDeploymentService { return &APIDeploymentService{ - store: store, - db: db, - snapshotManager: snapshotManager, - parser: config.NewParser(), - validator: validator, - httpClient: &http.Client{Timeout: 10 * time.Second}, - routerConfig: routerConfig, - eventHub: eventHub, - enableMultiTenantMode: enableMultiTenantMode, + store: store, + db: db, + snapshotManager: snapshotManager, + parser: config.NewParser(), + validator: validator, + httpClient: &http.Client{Timeout: 10 * time.Second}, + routerConfig: routerConfig, + eventHub: eventHub, + enableReplicaSync: enableReplicaSync, } } @@ -276,7 +276,7 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams } // Update xDS snapshot or publish event based on multi-tenant mode - if !s.enableMultiTenantMode { + if !s.enableReplicaSync { // Single-tenant mode: Update xDS snapshot asynchronously go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -393,7 +393,7 @@ func (s *APIDeploymentService) saveOrUpdateConfig(storedCfg *models.StoredConfig } // Try to add to in-memory store - if !s.enableMultiTenantMode { + if !s.enableReplicaSync { if err := s.store.Add(storedCfg); err != nil { // Check if it's a conflict (API already exists) if storage.IsConflictError(err) { From 247e2f73d92d3bd75eb3cb2ec8335f74f02d78a0 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Mon, 5 Jan 2026 09:43:28 +0530 Subject: [PATCH 13/24] Start Event Listener --- .../gateway-controller/cmd/controller/main.go | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 5e956a0e2..77f7bfd20 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -22,6 +22,7 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/controlplane" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/logger" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" @@ -119,6 +120,7 @@ func main() { if err := eventHub.Initialize(ctx); err != nil { log.Fatal("Failed to initialize EventHub", zap.Error(err)) } + eventHub.RegisterOrganization("default") log.Info("EventHub initialized successfully") } else { log.Fatal("EventHub requires persistent storage. Multi-tenant mode will not function correctly in memory-only mode.") @@ -264,6 +266,26 @@ func main() { log.Info("Policy xDS server is disabled") } + // Initialize and start EventListener if EventHub is available + var evtListener *eventlistener.EventListener + if eventHub != nil { + log.Info("Initializing EventListener") + eventSource := eventlistener.NewEventHubAdapter(eventHub, log) + evtListener = eventlistener.NewEventListener( + eventSource, + configStore, + db, + snapshotManager, + policyManager, // Can be nil if policy server is disabled + &cfg.GatewayController.Router, + log, + ) + if err := evtListener.Start(context.Background()); err != nil { + log.Fatal("Failed to start EventListener", zap.Error(err)) + } + log.Info("EventListener started successfully") + } + // Load policy definitions from files (must be done before creating validator) policyLoader := utils.NewPolicyLoader(log) policyDir := cfg.GatewayController.Policies.DefinitionsPath @@ -365,6 +387,11 @@ func main() { policyXDSServer.Stop() } + // Stop EventListener if it was started + if evtListener != nil { + evtListener.Stop() + } + log.Info("Gateway-Controller stopped") } From e197da5adaf87b639081f457de532aa955015bc8 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Mon, 5 Jan 2026 09:45:46 +0530 Subject: [PATCH 14/24] Refactor - Organization ID --- .../pkg/eventhub/backend.go | 8 +- .../pkg/eventhub/eventhub.go | 6 +- .../pkg/eventhub/eventhub_test.go | 2 +- .../pkg/eventhub/sqlite_backend.go | 12 +-- .../gateway-controller/pkg/eventhub/topic.go | 14 ++-- .../gateway-controller/pkg/eventhub/types.go | 12 ++- .../pkg/eventlistener/eventhub_adapter.go | 6 +- .../pkg/storage/gateway-controller-db.sql | 77 +++++-------------- .../pkg/utils/api_deployment.go | 2 +- 9 files changed, 47 insertions(+), 92 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/backend.go b/gateway/gateway-controller/pkg/eventhub/backend.go index 94afc3167..b83165784 100644 --- a/gateway/gateway-controller/pkg/eventhub/backend.go +++ b/gateway/gateway-controller/pkg/eventhub/backend.go @@ -18,20 +18,20 @@ type EventhubImpl interface { // For SQLite: creates entry in organization_states table // For NATS: creates subject/stream for the organization // For Azure Service Bus: creates topic for the organization - RegisterOrganization(ctx context.Context, orgID OrganizationID) error + RegisterOrganization(ctx context.Context, orgID string) error // Publish publishes an event for an organization. // The implementation should ensure delivery semantics appropriate for the broker. - Publish(ctx context.Context, orgID OrganizationID, eventType EventType, + Publish(ctx context.Context, orgID string, eventType EventType, action, entityID string, eventData []byte) error // Subscribe registers a channel to receive events for an organization. // Events are delivered as batches (slices) when available. // The subscriber receives ALL event types and should filter if needed. - Subscribe(orgID OrganizationID, eventChan chan<- []Event) error + Subscribe(orgID string, eventChan chan<- []Event) error // Unsubscribe removes a subscription channel for an organization. - Unsubscribe(orgID OrganizationID, eventChan chan<- []Event) error + Unsubscribe(orgID string, eventChan chan<- []Event) error // Cleanup removes old events based on retention policy. // For SQLite: deletes events older than specified time diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index 3f2fc351f..0f44f77be 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -59,19 +59,19 @@ func (eh *eventHub) Initialize(ctx context.Context) error { } // RegisterOrganization registers a new organization with the EventHub -func (eh *eventHub) RegisterOrganization(organizationID OrganizationID) error { +func (eh *eventHub) RegisterOrganization(organizationID string) error { ctx := context.Background() return eh.backend.RegisterOrganization(ctx, organizationID) } // PublishEvent publishes an event for an organization -func (eh *eventHub) PublishEvent(ctx context.Context, organizationID OrganizationID, +func (eh *eventHub) PublishEvent(ctx context.Context, organizationID string, eventType EventType, action, entityID string, eventData []byte) error { return eh.backend.Publish(ctx, organizationID, eventType, action, entityID, eventData) } // Subscribe registers a channel to receive events for an organization -func (eh *eventHub) Subscribe(organizationID OrganizationID, eventChan chan<- []Event) error { +func (eh *eventHub) Subscribe(organizationID string, eventChan chan<- []Event) error { return eh.backend.Subscribe(organizationID, eventChan) } diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go index 1010d323e..6998dbdd5 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go @@ -95,7 +95,7 @@ func TestEventHub_PublishAndSubscribe(t *testing.T) { select { case events := <-eventChan: assert.GreaterOrEqual(t, len(events), 1) - assert.Equal(t, OrganizationID("test-org"), events[0].OrganizationID) + assert.Equal(t, "test-org", events[0].OrganizationID) assert.Equal(t, EventTypeAPI, events[0].EventType) assert.Equal(t, "CREATE", events[0].Action) assert.Equal(t, "api-1", events[0].EntityID) diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 186c882be..448571cec 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -73,7 +73,7 @@ func (b *SQLiteBackend) Initialize(ctx context.Context) error { } // RegisterOrganization creates the necessary resources for an organization -func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID OrganizationID) error { +func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID string) error { // Register in local registry if err := b.registry.register(orgID); err != nil { return err @@ -98,7 +98,7 @@ func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID Organiza } // Publish publishes an event for an organization -func (b *SQLiteBackend) Publish(ctx context.Context, orgID OrganizationID, +func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, eventType EventType, action, entityID string, eventData []byte) error { // Verify organization is registered @@ -157,7 +157,7 @@ func (b *SQLiteBackend) Publish(ctx context.Context, orgID OrganizationID, } // Subscribe registers a channel to receive events for an organization -func (b *SQLiteBackend) Subscribe(orgID OrganizationID, eventChan chan<- []Event) error { +func (b *SQLiteBackend) Subscribe(orgID string, eventChan chan<- []Event) error { if err := b.registry.addSubscriber(orgID, eventChan); err != nil { return err } @@ -169,7 +169,7 @@ func (b *SQLiteBackend) Subscribe(orgID OrganizationID, eventChan chan<- []Event } // Unsubscribe removes a subscription channel for an organization -func (b *SQLiteBackend) Unsubscribe(orgID OrganizationID, eventChan chan<- []Event) error { +func (b *SQLiteBackend) Unsubscribe(orgID string, eventChan chan<- []Event) error { if err := b.registry.removeSubscriber(orgID, eventChan); err != nil { return err } @@ -275,7 +275,7 @@ func (b *SQLiteBackend) pollAllOrganizations() { // Check each organization for changes for _, state := range states { - orgID := OrganizationID(state.Organization) + orgID := state.Organization org, err := b.registry.get(orgID) if err != nil { @@ -337,7 +337,7 @@ func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, } // getEventsSince retrieves events for an organization after a given timestamp -func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID OrganizationID, since time.Time) ([]Event, error) { +func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since time.Time) ([]Event, error) { query := ` SELECT processed_timestamp, originated_timestamp, event_type, action, entity_id, event_data diff --git a/gateway/gateway-controller/pkg/eventhub/topic.go b/gateway/gateway-controller/pkg/eventhub/topic.go index 824b66f25..459a0db93 100644 --- a/gateway/gateway-controller/pkg/eventhub/topic.go +++ b/gateway/gateway-controller/pkg/eventhub/topic.go @@ -13,7 +13,7 @@ var ( // organization represents an internal organization with its subscriptions and poll state type organization struct { - id OrganizationID + id string subscribers []chan<- []Event // Registered subscription channels subscriberMu sync.RWMutex @@ -24,19 +24,19 @@ type organization struct { // organizationRegistry manages all registered organizations type organizationRegistry struct { - orgs map[OrganizationID]*organization + orgs map[string]*organization mu sync.RWMutex } // newOrganizationRegistry creates a new organization registry func newOrganizationRegistry() *organizationRegistry { return &organizationRegistry{ - orgs: make(map[OrganizationID]*organization), + orgs: make(map[string]*organization), } } // register adds a new organization to the registry -func (r *organizationRegistry) register(id OrganizationID) error { +func (r *organizationRegistry) register(id string) error { r.mu.Lock() defer r.mu.Unlock() @@ -54,7 +54,7 @@ func (r *organizationRegistry) register(id OrganizationID) error { } // get retrieves an organization by ID -func (r *organizationRegistry) get(id OrganizationID) (*organization, error) { +func (r *organizationRegistry) get(id string) (*organization, error) { r.mu.RLock() defer r.mu.RUnlock() @@ -66,7 +66,7 @@ func (r *organizationRegistry) get(id OrganizationID) (*organization, error) { } // addSubscriber adds a subscription channel to an organization -func (r *organizationRegistry) addSubscriber(id OrganizationID, ch chan<- []Event) error { +func (r *organizationRegistry) addSubscriber(id string, ch chan<- []Event) error { r.mu.RLock() org, exists := r.orgs[id] r.mu.RUnlock() @@ -82,7 +82,7 @@ func (r *organizationRegistry) addSubscriber(id OrganizationID, ch chan<- []Even } // removeSubscriber removes a subscription channel from an organization -func (r *organizationRegistry) removeSubscriber(id OrganizationID, ch chan<- []Event) error { +func (r *organizationRegistry) removeSubscriber(id string, ch chan<- []Event) error { r.mu.RLock() org, exists := r.orgs[id] r.mu.RUnlock() diff --git a/gateway/gateway-controller/pkg/eventhub/types.go b/gateway/gateway-controller/pkg/eventhub/types.go index cf183e5fa..393f8a993 100644 --- a/gateway/gateway-controller/pkg/eventhub/types.go +++ b/gateway/gateway-controller/pkg/eventhub/types.go @@ -5,9 +5,6 @@ import ( "time" ) -// OrganizationID represents a unique organization identifier -type OrganizationID string - // EventType represents the type of event type EventType string @@ -20,7 +17,7 @@ const ( // Event represents a single event in the hub type Event struct { - OrganizationID OrganizationID // Organization this event belongs to + OrganizationID string // Organization this event belongs to ProcessedTimestamp time.Time // When event was recorded in DB OriginatedTimestamp time.Time // When event was created EventType EventType // Type of event (API, CERTIFICATE, etc.) @@ -43,17 +40,17 @@ type EventHub interface { // RegisterOrganization registers an organization for event tracking // Creates entry in organization_states table with empty version - RegisterOrganization(organizationID OrganizationID) error + RegisterOrganization(organizationID string) error // PublishEvent publishes an event for an organization // Updates the organization_states and events tables atomically - PublishEvent(ctx context.Context, organizationID OrganizationID, eventType EventType, + PublishEvent(ctx context.Context, organizationID string, eventType EventType, action, entityID string, eventData []byte) error // Subscribe registers a channel to receive events for an organization // Events are delivered as batches (arrays) based on poll cycle // Subscriber receives ALL event types and should filter by EventType if needed - Subscribe(organizationID OrganizationID, eventChan chan<- []Event) error + Subscribe(organizationID string, eventChan chan<- []Event) error // CleanUpEvents removes events between the specified time range CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Time) error @@ -62,6 +59,7 @@ type EventHub interface { Close() error } + // Config holds EventHub configuration type Config struct { // PollInterval is how often to poll for state changes diff --git a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go index e5d20a89d..734e07a84 100644 --- a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go +++ b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go @@ -62,10 +62,8 @@ func (a *EventHubAdapter) Subscribe(ctx context.Context, organizationID string, return fmt.Errorf("already subscribed to organization: %s", organizationID) } - orgID := eventhub.OrganizationID(organizationID) - // Register organization with EventHub (idempotent operation) - if err := a.eventHub.RegisterOrganization(orgID); err != nil { + if err := a.eventHub.RegisterOrganization(organizationID); err != nil { a.logger.Debug("Organization may already be registered", zap.String("organization", organizationID), zap.Error(err), @@ -77,7 +75,7 @@ func (a *EventHubAdapter) Subscribe(ctx context.Context, organizationID string, bridgeChan := make(chan []eventhub.Event, 10) // Subscribe to EventHub - if err := a.eventHub.Subscribe(orgID, bridgeChan); err != nil { + if err := a.eventHub.Subscribe(organizationID, bridgeChan); err != nil { close(bridgeChan) return fmt.Errorf("failed to subscribe to eventhub: %w", err) } diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index bbf3a5976..962d37859 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -150,72 +150,31 @@ CREATE INDEX IF NOT EXISTS idx_api_key_status ON api_keys(status); CREATE INDEX IF NOT EXISTS idx_api_key_expiry ON api_keys(expires_at); CREATE INDEX IF NOT EXISTS idx_created_by ON api_keys(created_by); --- API Events table (added in schema version 6) --- Stores events for API entity changes -CREATE TABLE IF NOT EXISTS api_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - originated_timestamp TIMESTAMP NOT NULL, - organization_id TEXT NOT NULL DEFAULT 'default', - action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), - entity_id TEXT NOT NULL, - event_data TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_api_events_lookup ON api_events(organization_id, processed_timestamp); - --- Certificate Events table (added in schema version 6) --- Stores events for certificate entity changes -CREATE TABLE IF NOT EXISTS certificate_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - originated_timestamp TIMESTAMP NOT NULL, - organization_id TEXT NOT NULL DEFAULT 'default', - action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), - entity_id TEXT NOT NULL, - event_data TEXT NOT NULL +-- EventHub: Organization States Table (added in schema version 9) +-- Tracks version information per organization for change detection +CREATE TABLE IF NOT EXISTS organization_states ( + organization TEXT PRIMARY KEY, + version_id TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX IF NOT EXISTS idx_cert_events_lookup ON certificate_events(organization_id, processed_timestamp); +CREATE INDEX IF NOT EXISTS idx_organization_states_updated ON organization_states(updated_at); --- LLM Template Events table (added in schema version 6) --- Stores events for LLM template entity changes -CREATE TABLE IF NOT EXISTS llm_template_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, +-- EventHub: Unified Events Table (added in schema version 9) +-- Stores all entity change events (APIs, certificates, LLM templates, etc.) +CREATE TABLE IF NOT EXISTS events ( + organization_id TEXT NOT NULL, processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, originated_timestamp TIMESTAMP NOT NULL, - organization_id TEXT NOT NULL DEFAULT 'default', + event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, - event_data TEXT NOT NULL + event_data TEXT NOT NULL, + PRIMARY KEY (organization_id, processed_timestamp) ); -CREATE INDEX IF NOT EXISTS idx_llm_events_lookup ON llm_template_events(organization_id, processed_timestamp); - --- EventHub: Topic States Table (added in schema version 7) --- Tracks version information per topic for change detection -CREATE TABLE IF NOT EXISTS topic_states ( - organization TEXT NOT NULL, - topic_name TEXT NOT NULL, - version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (organization, topic_name) -); +CREATE INDEX IF NOT EXISTS idx_events_lookup ON events(organization_id, processed_timestamp); +CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type); -CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at); - --- EventHub: Example events table template --- Each topic needs its own events table following this pattern: --- Table name format: {topic_name}_events --- --- Example for 'api' topic: --- CREATE TABLE IF NOT EXISTS api_events ( --- id INTEGER PRIMARY KEY AUTOINCREMENT, --- processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, --- originated_timestamp TIMESTAMP NOT NULL, --- event_data TEXT NOT NULL --- ); --- CREATE INDEX IF NOT EXISTS idx_api_events_processed ON api_events(processed_timestamp); - --- Set schema version to 7 -PRAGMA user_version = 8; +-- Set schema version to 9 +PRAGMA user_version = 9; diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index ffe2ea1f3..07925d4af 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -299,7 +299,7 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams } // Use default organization ID (can be made configurable in future) - organizationID := eventhub.OrganizationID("default") + organizationID := "default" // Publish event with empty payload as per requirements ctx := context.Background() From ab7ef782a9e9bb1118551c217803b91d4de84055 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Mon, 5 Jan 2026 13:19:23 +0530 Subject: [PATCH 15/24] Include correlation ID for tracing event processing --- .../pkg/eventhub/backend.go | 2 +- .../pkg/eventhub/eventhub.go | 4 ++-- .../pkg/eventhub/eventhub_test.go | 17 ++++++++------- .../pkg/eventhub/sqlite_backend.go | 21 ++++++++++--------- .../gateway-controller/pkg/eventhub/types.go | 17 ++++++++------- .../pkg/eventlistener/api_processor.go | 14 ++++++------- .../pkg/eventlistener/event_source.go | 3 +++ .../pkg/eventlistener/eventhub_adapter.go | 1 + .../pkg/eventlistener/listener.go | 1 + .../pkg/storage/gateway-controller-db.sql | 1 + .../gateway-controller/pkg/storage/sqlite.go | 1 + .../pkg/utils/api_deployment.go | 2 +- 12 files changed, 46 insertions(+), 38 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/backend.go b/gateway/gateway-controller/pkg/eventhub/backend.go index b83165784..108d66ece 100644 --- a/gateway/gateway-controller/pkg/eventhub/backend.go +++ b/gateway/gateway-controller/pkg/eventhub/backend.go @@ -23,7 +23,7 @@ type EventhubImpl interface { // Publish publishes an event for an organization. // The implementation should ensure delivery semantics appropriate for the broker. Publish(ctx context.Context, orgID string, eventType EventType, - action, entityID string, eventData []byte) error + action, entityID, correlationID string, eventData []byte) error // Subscribe registers a channel to receive events for an organization. // Events are delivered as batches (slices) when available. diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index 0f44f77be..f31573c78 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -66,8 +66,8 @@ func (eh *eventHub) RegisterOrganization(organizationID string) error { // PublishEvent publishes an event for an organization func (eh *eventHub) PublishEvent(ctx context.Context, organizationID string, - eventType EventType, action, entityID string, eventData []byte) error { - return eh.backend.Publish(ctx, organizationID, eventType, action, entityID, eventData) + eventType EventType, action, entityID, correlationID string, eventData []byte) error { + return eh.backend.Publish(ctx, organizationID, eventType, action, entityID, correlationID, eventData) } // Subscribe registers a channel to receive events for an organization diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go index 6998dbdd5..54eaaf909 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub_test.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub_test.go @@ -36,6 +36,7 @@ func setupTestDB(t *testing.T) *sql.DB { event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, + correlation_id TEXT NOT NULL DEFAULT '', event_data TEXT NOT NULL, PRIMARY KEY (organization_id, processed_timestamp) ) @@ -88,7 +89,7 @@ func TestEventHub_PublishAndSubscribe(t *testing.T) { // Publish event data, _ := json.Marshal(map[string]string{"key": "value"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id", data) require.NoError(t, err) // Wait for event delivery via polling @@ -121,7 +122,7 @@ func TestEventHub_CleanUpEvents(t *testing.T) { // Publish events for i := 0; i < 5; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id", data) require.NoError(t, err) } @@ -159,7 +160,7 @@ func TestEventHub_PollerDetectsChanges(t *testing.T) { // Publish multiple events for i := 0; i < 3; i++ { data, _ := json.Marshal(map[string]int{"index": i}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id", data) require.NoError(t, err) time.Sleep(10 * time.Millisecond) } @@ -198,7 +199,7 @@ func TestEventHub_AtomicPublish(t *testing.T) { // Publish event data, _ := json.Marshal(map[string]string{"test": "data"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id", data) require.NoError(t, err) // Verify event was recorded in unified table @@ -240,7 +241,7 @@ func TestEventHub_MultipleSubscribers(t *testing.T) { // Publish event data, _ := json.Marshal(map[string]string{"test": "multi"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id", data) require.NoError(t, err) // Both subscribers should receive the event @@ -305,15 +306,15 @@ func TestEventHub_MultipleEventTypes(t *testing.T) { // Publish different event types data1, _ := json.Marshal(map[string]string{"type": "api"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", data1) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeAPI, "CREATE", "api-1", "test-correlation-id-1", data1) require.NoError(t, err) data2, _ := json.Marshal(map[string]string{"type": "cert"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeCertificate, "UPDATE", "cert-1", data2) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeCertificate, "UPDATE", "cert-1", "test-correlation-id-2", data2) require.NoError(t, err) data3, _ := json.Marshal(map[string]string{"type": "llm"}) - err = hub.PublishEvent(context.Background(), "test-org", EventTypeLLMTemplate, "DELETE", "template-1", data3) + err = hub.PublishEvent(context.Background(), "test-org", EventTypeLLMTemplate, "DELETE", "template-1", "test-correlation-id-3", data3) require.NoError(t, err) // Wait for events to be delivered (all types should come through) diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 448571cec..9b61dc8d9 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -19,11 +19,11 @@ type SQLiteBackend struct { registry *organizationRegistry // Polling state - pollerCtx context.Context - pollerCancel context.CancelFunc - cleanupCtx context.Context + pollerCtx context.Context + pollerCancel context.CancelFunc + cleanupCtx context.Context cleanupCancel context.CancelFunc - wg sync.WaitGroup + wg sync.WaitGroup initialized bool mu sync.RWMutex @@ -99,7 +99,7 @@ func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID string) // Publish publishes an event for an organization func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, - eventType EventType, action, entityID string, eventData []byte) error { + eventType EventType, action, entityID, correlationID string, eventData []byte) error { // Verify organization is registered _, err := b.registry.get(orgID) @@ -119,11 +119,11 @@ func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, // Insert event insertQuery := ` INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, - event_type, action, entity_id, event_data) - VALUES (?, ?, ?, ?, ?, ?, ?) + event_type, action, entity_id, correlation_id, event_data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ` _, err = tx.ExecContext(ctx, insertQuery, - string(orgID), now, now, string(eventType), action, entityID, eventData) + string(orgID), now, now, string(eventType), action, entityID, correlationID, eventData) if err != nil { return fmt.Errorf("failed to record event: %w", err) } @@ -150,6 +150,7 @@ func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, zap.String("eventType", string(eventType)), zap.String("action", action), zap.String("entityID", entityID), + zap.String("correlationID", correlationID), zap.String("version", newVersion), ) @@ -340,7 +341,7 @@ func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since time.Time) ([]Event, error) { query := ` SELECT processed_timestamp, originated_timestamp, event_type, - action, entity_id, event_data + action, entity_id, correlation_id, event_data FROM events WHERE organization_id = ? AND processed_timestamp > ? ORDER BY processed_timestamp ASC @@ -359,7 +360,7 @@ func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since e.OrganizationID = orgID if err := rows.Scan(&e.ProcessedTimestamp, &e.OriginatedTimestamp, - &eventTypeStr, &e.Action, &e.EntityID, &e.EventData); err != nil { + &eventTypeStr, &e.Action, &e.EntityID, &e.CorrelationID, &e.EventData); err != nil { return nil, fmt.Errorf("failed to scan event: %w", err) } e.EventType = EventType(eventTypeStr) diff --git a/gateway/gateway-controller/pkg/eventhub/types.go b/gateway/gateway-controller/pkg/eventhub/types.go index 393f8a993..283d41558 100644 --- a/gateway/gateway-controller/pkg/eventhub/types.go +++ b/gateway/gateway-controller/pkg/eventhub/types.go @@ -17,13 +17,14 @@ const ( // Event represents a single event in the hub type Event struct { - OrganizationID string // Organization this event belongs to - ProcessedTimestamp time.Time // When event was recorded in DB - OriginatedTimestamp time.Time // When event was created - EventType EventType // Type of event (API, CERTIFICATE, etc.) - Action string // CREATE, UPDATE, or DELETE - EntityID string // ID of the affected entity - EventData []byte // JSON serialized payload + OrganizationID string // Organization this event belongs to + ProcessedTimestamp time.Time // When event was recorded in DB + OriginatedTimestamp time.Time // When event was created + EventType EventType // Type of event (API, CERTIFICATE, etc.) + Action string // CREATE, UPDATE, or DELETE + EntityID string // ID of the affected entity + CorrelationID string // Correlation ID for request tracing + EventData []byte // JSON serialized payload } // OrganizationState represents the version state for an organization @@ -45,7 +46,7 @@ type EventHub interface { // PublishEvent publishes an event for an organization // Updates the organization_states and events tables atomically PublishEvent(ctx context.Context, organizationID string, eventType EventType, - action, entityID string, eventData []byte) error + action, entityID, correlationID string, eventData []byte) error // Subscribe registers a channel to receive events for an organization // Events are delivered as batches (arrays) based on poll cycle diff --git a/gateway/gateway-controller/pkg/eventlistener/api_processor.go b/gateway/gateway-controller/pkg/eventlistener/api_processor.go index 74b65a8f4..c2a44bf53 100644 --- a/gateway/gateway-controller/pkg/eventlistener/api_processor.go +++ b/gateway/gateway-controller/pkg/eventlistener/api_processor.go @@ -20,7 +20,6 @@ package eventlistener import ( "context" - "fmt" "strings" "time" @@ -37,22 +36,23 @@ func (el *EventListener) processAPIEvents(event Event) { log := el.logger.With( zap.String("api_id", event.EntityID), zap.String("action", event.Action), + zap.String("correlation_id", event.CorrelationID), ) apiID := event.EntityID - + // TODO: (VirajSalaka) Use Context to propogate correlationID switch event.Action { case "CREATE", "UPDATE": - el.handleAPICreateOrUpdate(apiID, log) + el.handleAPICreateOrUpdate(apiID, event.CorrelationID, log) case "DELETE": - el.handleAPIDelete(apiID, log) + el.handleAPIDelete(apiID, event.CorrelationID, log) default: log.Warn("Unknown action type") } } // handleAPICreateOrUpdate fetches the API from DB and updates XDS -func (el *EventListener) handleAPICreateOrUpdate(apiID string, log *zap.Logger) { +func (el *EventListener) handleAPICreateOrUpdate(apiID string, correlationID string, log *zap.Logger) { // 1. Fetch API configuration from database config, err := el.db.GetConfig(apiID) if err != nil { @@ -85,7 +85,6 @@ func (el *EventListener) handleAPICreateOrUpdate(apiID string, log *zap.Logger) } // 3. Trigger async XDS snapshot update - correlationID := fmt.Sprintf("event-%s-%d", apiID, time.Now().UnixNano()) go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -108,7 +107,7 @@ func (el *EventListener) handleAPICreateOrUpdate(apiID string, log *zap.Logger) } // handleAPIDelete removes the API from in-memory store and updates XDS -func (el *EventListener) handleAPIDelete(apiID string, log *zap.Logger) { +func (el *EventListener) handleAPIDelete(apiID string, correlationID string, log *zap.Logger) { // 1. Check if config exists in store (for logging/policy removal) config, err := el.store.Get(apiID) if err != nil { @@ -125,7 +124,6 @@ func (el *EventListener) handleAPIDelete(apiID string, log *zap.Logger) { } // 3. Trigger async XDS snapshot update - correlationID := fmt.Sprintf("event-delete-%s-%d", apiID, time.Now().UnixNano()) go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/gateway/gateway-controller/pkg/eventlistener/event_source.go b/gateway/gateway-controller/pkg/eventlistener/event_source.go index 269214a5d..0dfbbc759 100644 --- a/gateway/gateway-controller/pkg/eventlistener/event_source.go +++ b/gateway/gateway-controller/pkg/eventlistener/event_source.go @@ -39,6 +39,9 @@ type Event struct { // EntityID identifies the specific entity affected by the event EntityID string + // CorrelationID is used to trace requests across services + CorrelationID string + // EventData contains the serialized event payload (typically JSON) EventData []byte diff --git a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go index 734e07a84..c24ae0abd 100644 --- a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go +++ b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go @@ -133,6 +133,7 @@ func (a *EventHubAdapter) bridgeEvents( Action: hubEvent.Action, EntityID: hubEvent.EntityID, EventData: hubEvent.EventData, + CorrelationID: hubEvent.CorrelationID, Timestamp: hubEvent.ProcessedTimestamp, } } diff --git a/gateway/gateway-controller/pkg/eventlistener/listener.go b/gateway/gateway-controller/pkg/eventlistener/listener.go index 97d3041a7..db9925e70 100644 --- a/gateway/gateway-controller/pkg/eventlistener/listener.go +++ b/gateway/gateway-controller/pkg/eventlistener/listener.go @@ -128,6 +128,7 @@ func (el *EventListener) handleEvent(event Event) { zap.String("event_type", event.EventType), zap.String("action", event.Action), zap.String("entity_id", event.EntityID), + zap.String("correlation_id", event.CorrelationID), ) switch event.EventType { diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index 962d37859..05d51c047 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -169,6 +169,7 @@ CREATE TABLE IF NOT EXISTS events ( event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, + correlation_id TEXT NOT NULL DEFAULT '', event_data TEXT NOT NULL, PRIMARY KEY (organization_id, processed_timestamp) ); diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index d732c4343..f70c82914 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -357,6 +357,7 @@ func (s *SQLiteStorage) initSchema() error { event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, + correlation_id TEXT NOT NULL DEFAULT '', event_data TEXT NOT NULL, PRIMARY KEY (organization_id, processed_timestamp) );`); err != nil { diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 07925d4af..d3ff92809 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -303,7 +303,7 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams // Publish event with empty payload as per requirements ctx := context.Background() - if err := s.eventHub.PublishEvent(ctx, organizationID, eventhub.EventTypeAPI, action, apiID, []byte{}); err != nil { + if err := s.eventHub.PublishEvent(ctx, organizationID, eventhub.EventTypeAPI, action, apiID, params.CorrelationID, []byte{}); err != nil { params.Logger.Error("Failed to publish event to eventhub", zap.Error(err), zap.String("api_id", apiID), From e615dd6c1d263e1d650cf64de1c5804b127e646b Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Tue, 6 Jan 2026 08:47:14 +0530 Subject: [PATCH 16/24] Move API Update related logic to api_deployment.go and reuse methods between createAPI and UpdateAPI --- .../gateway-controller/cmd/controller/main.go | 2 +- .../pkg/api/handlers/handlers.go | 491 ++------------- .../pkg/api/handlers/policy_ordering_test.go | 20 +- .../pkg/controlplane/client.go | 4 +- .../pkg/utils/api_deployment.go | 567 +++++++++++++++--- .../utils/websub_topic_registration_test.go | 8 +- 6 files changed, 560 insertions(+), 532 deletions(-) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 84f9ca684..5ebca1a72 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -272,7 +272,7 @@ func main() { validator.SetPolicyValidator(policyValidator) // Initialize and start control plane client with dependencies for API creation - cpClient := controlplane.NewClient(cfg.GatewayController.ControlPlane, log, configStore, db, snapshotManager, validator, &cfg.GatewayController.Router) + cpClient := controlplane.NewClient(cfg.GatewayController.ControlPlane, log, configStore, db, snapshotManager, policyManager, validator, &cfg.GatewayController.Router) if err := cpClient.Start(); err != nil { log.Error("Failed to start control plane client", zap.Error(err)) // Don't fail startup - gateway can run in degraded mode without control plane diff --git a/gateway/gateway-controller/pkg/api/handlers/handlers.go b/gateway/gateway-controller/pkg/api/handlers/handlers.go index 0634bd4ab..6828201c3 100644 --- a/gateway/gateway-controller/pkg/api/handlers/handlers.go +++ b/gateway/gateway-controller/pkg/api/handlers/handlers.go @@ -22,9 +22,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/wso2/api-platform/common/constants" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/apikeyxds" - "io" "net/http" "sort" @@ -34,9 +31,11 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/wso2/api-platform/common/constants" commonmodels "github.com/wso2/api-platform/common/models" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/middleware" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/apikeyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/controlplane" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" @@ -44,7 +43,6 @@ import ( "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/utils" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" - policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" "go.uber.org/zap" ) @@ -83,7 +81,7 @@ func NewAPIServer( routerConfig *config.RouterConfig, apiKeyXDSManager *apikeyxds.APIKeyStateManager, ) *APIServer { - deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig) + deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, policyManager, validator, routerConfig) server := &APIServer{ store: store, db: db, @@ -225,30 +223,7 @@ func (s *APIServer) CreateAPI(c *gin.Context) { Id: stringPtr(result.StoredConfig.GetHandle()), CreatedAt: timePtr(result.StoredConfig.CreatedAt), }) - - // Build and add policy config derived from API configuration if policies are present - if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(result.StoredConfig) - if storedPolicy != nil { - if err := s.policyManager.AddPolicy(storedPolicy); err != nil { - log.Error("Failed to add derived policy configuration", zap.Error(err)) - } else { - log.Info("Derived policy configuration added", - zap.String("policy_id", storedPolicy.ID), - zap.Int("route_count", len(storedPolicy.Configuration.Routes))) - } - } else if result.IsUpdate { - // API was updated and no longer has policies, remove the existing policy configuration - policyID := result.StoredConfig.ID + "-policies" - if err := s.policyManager.RemovePolicy(policyID); err != nil { - // Log at debug level since policy may not exist if API never had policies - log.Debug("No policy configuration to remove", zap.String("policy_id", policyID)) - } else { - log.Info("Derived policy configuration removed (API no longer has policies)", - zap.String("policy_id", policyID)) - } - } - } + // Policy management is now handled by the deployment service } // ListAPIs implements ServerInterface.ListAPIs @@ -487,7 +462,6 @@ func (s *APIServer) GetAPIById(c *gin.Context, id string) { func (s *APIServer) UpdateAPI(c *gin.Context, id string) { // Get correlation-aware logger from context log := middleware.GetLogger(c, s.logger) - handle := id // Read request body body, err := io.ReadAll(c.Request.Body) @@ -500,249 +474,82 @@ func (s *APIServer) UpdateAPI(c *gin.Context, id string) { return } - // Parse configuration - contentType := c.GetHeader("Content-Type") - var apiConfig api.APIConfiguration - err = s.parser.Parse(body, contentType, &apiConfig) - if err != nil { - log.Error("Failed to parse configuration", zap.Error(err)) - c.JSON(http.StatusBadRequest, api.ErrorResponse{ - Status: "error", - Message: "Failed to parse configuration", - }) - return - } + // Get correlation ID from context + correlationID := middleware.GetCorrelationID(c) - // Validate that the handle in the YAML matches the path parameter - if apiConfig.Metadata.Name != "" { - if apiConfig.Metadata.Name != handle { - log.Warn("Handle mismatch between path and YAML metadata", - zap.String("path_handle", handle), - zap.String("yaml_handle", apiConfig.Metadata.Name)) + // Update API configuration using the utility service + result, err := s.deploymentService.UpdateAPIConfiguration(utils.APIUpdateParams{ + Handle: id, + Data: body, + ContentType: c.GetHeader("Content-Type"), + CorrelationID: correlationID, + Logger: log, + }) + + if err != nil { + // Map error types to HTTP status codes + switch e := err.(type) { + case *utils.NotFoundError: + c.JSON(http.StatusNotFound, api.ErrorResponse{ + Status: "error", + Message: e.Error(), + }) + case *utils.HandleMismatchError: c.JSON(http.StatusBadRequest, api.ErrorResponse{ Status: "error", - Message: fmt.Sprintf("Handle mismatch: path has '%s' but YAML metadata.name has '%s'", handle, apiConfig.Metadata.Name), + Message: e.Error(), }) - return - } - } - - // Validate configuration - validationErrors := s.validator.Validate(&apiConfig) - if len(validationErrors) > 0 { - log.Warn("Configuration validation failed", - zap.String("handle", handle), - zap.Int("num_errors", len(validationErrors))) - - errors := make([]api.ValidationError, len(validationErrors)) - for i, e := range validationErrors { - errors[i] = api.ValidationError{ - Field: stringPtr(e.Field), - Message: stringPtr(e.Message), - } - } - - c.JSON(http.StatusBadRequest, api.ErrorResponse{ - Status: "error", - Message: "Configuration validation failed", - Errors: &errors, - }) - return - } - - if s.db == nil { - c.JSON(http.StatusServiceUnavailable, api.ErrorResponse{ - Status: "error", - Message: "Database storage not available", - }) - return - } - - // Check if config exists - existing, err := s.db.GetConfigByHandle(handle) - if err != nil { - log.Warn("API configuration not found", - zap.String("handle", handle)) - c.JSON(http.StatusNotFound, api.ErrorResponse{ - Status: "error", - Message: fmt.Sprintf("API configuration with handle '%s' not found", handle), - }) - return - } - - // Update stored configuration - now := time.Now() - existing.Configuration = apiConfig - existing.Status = models.StatusPending - existing.UpdatedAt = now - existing.DeployedAt = nil - existing.DeployedVersion = 0 - - if apiConfig.Kind == api.Asyncwebsub { - topicsToRegister, topicsToUnregister := s.deploymentService.GetTopicsForUpdate(*existing) - // TODO: Pre configure the dynamic forward proxy rules for event gw - // This was communication bridge will be created on the gw startup - // Can perform internal communication with websub hub without relying on the dynamic rules - // Execute topic operations with wait group and errors tracking - var wg2 sync.WaitGroup - var regErrs int32 - var deregErrs int32 - - if len(topicsToRegister) > 0 { - wg2.Add(1) - go func(list []string) { - defer wg2.Done() - log.Info("Starting topic registration", zap.Int("total_topics", len(list)), zap.String("api_id", existing.ID)) - //fmt.Println("Topics Registering Started") - var childWg sync.WaitGroup - for _, topic := range list { - childWg.Add(1) - go func(topic string) { - defer childWg.Done() - if err := s.deploymentService.RegisterTopicWithHub(s.httpClient, topic, "localhost", 8083, log); err != nil { - log.Error("Failed to register topic with WebSubHub", - zap.Error(err), - zap.String("topic", topic), - zap.String("api_id", existing.ID)) - atomic.AddInt32(®Errs, 1) - } else { - log.Info("Successfully registered topic with WebSubHub", - zap.String("topic", topic), - zap.String("api_id", existing.ID)) - } - }(topic) - } - childWg.Wait() - }(topicsToRegister) - } - - if len(topicsToUnregister) > 0 { - wg2.Add(1) - go func(list []string) { - defer wg2.Done() - log.Info("Starting topic deregistration", zap.Int("total_topics", len(list)), zap.String("api_id", existing.ID)) - var childWg sync.WaitGroup - for _, topic := range list { - childWg.Add(1) - go func(topic string) { - defer childWg.Done() - if err := s.deploymentService.UnregisterTopicWithHub(s.httpClient, topic, "localhost", 8083, log); err != nil { - log.Error("Failed to deregister topic from WebSubHub", - zap.Error(err), - zap.String("topic", topic), - zap.String("api_id", existing.ID)) - atomic.AddInt32(&deregErrs, 1) - } else { - log.Info("Successfully deregistered topic from WebSubHub", - zap.String("topic", topic), - zap.String("api_id", existing.ID)) - } - }(topic) + case *utils.APIValidationError: + errors := make([]api.ValidationError, len(e.Errors)) + for i, ve := range e.Errors { + errors[i] = api.ValidationError{ + Field: stringPtr(ve.Field), + Message: stringPtr(ve.Message), } - childWg.Wait() - }(topicsToUnregister) - } - wg2.Wait() - - log.Info("Topic lifecycle operations completed", - zap.String("api_id", existing.ID), - zap.Int("registered", len(topicsToRegister)), - zap.Int("deregistered", len(topicsToUnregister)), - zap.Int("register_errors", int(regErrs)), - zap.Int("deregister_errors", int(deregErrs))) - - // Check if topic operations failed and return error - if regErrs > 0 || deregErrs > 0 { - log.Error("Failed to register & deregister topics", zap.Error(err)) - c.JSON(http.StatusInternalServerError, api.ErrorResponse{ + } + c.JSON(http.StatusBadRequest, api.ErrorResponse{ Status: "error", - Message: "Topic lifecycle operations failed", + Message: "Configuration validation failed", + Errors: &errors, }) - return - } - } - - // Atomic dual-write: database + in-memory - // Update database first (only if persistent mode) - if s.db != nil { - if err := s.db.UpdateConfig(existing); err != nil { - log.Error("Failed to update config in database", zap.Error(err)) + case *utils.ParseError: + c.JSON(http.StatusBadRequest, api.ErrorResponse{ + Status: "error", + Message: e.Error(), + }) + case *utils.TopicOperationError: c.JSON(http.StatusInternalServerError, api.ErrorResponse{ Status: "error", - Message: "Failed to persist configuration update", + Message: e.Error(), }) - return - } - } - - if err := s.store.Update(existing); err != nil { - // Log conflict errors at info level, other errors at error level - if storage.IsConflictError(err) { - log.Info("API configuration handle already exists", - zap.String("id", existing.ID), - zap.String("handle", handle)) + case *utils.ConflictError: c.JSON(http.StatusConflict, api.ErrorResponse{ Status: "error", - Message: err.Error(), + Message: e.Error(), }) - } else { - log.Error("Failed to update config in memory store", zap.Error(err)) + case *utils.DatabaseUnavailableError: + c.JSON(http.StatusServiceUnavailable, api.ErrorResponse{ + Status: "error", + Message: e.Error(), + }) + default: + log.Error("Failed to update API configuration", zap.Error(err)) c.JSON(http.StatusInternalServerError, api.ErrorResponse{ Status: "error", - Message: "Failed to update configuration in memory store", + Message: err.Error(), }) } return } - // Get correlation ID from context - correlationID := middleware.GetCorrelationID(c) - - // Update xDS snapshot asynchronously - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - if err := s.snapshotManager.UpdateSnapshot(ctx, correlationID); err != nil { - log.Error("Failed to update xDS snapshot", zap.Error(err)) - } - }() - - log.Info("API configuration updated", - zap.String("id", existing.ID), - zap.String("handle", handle)) - // Return success response (id is the handle) c.JSON(http.StatusOK, api.APIUpdateResponse{ Status: stringPtr("success"), Message: stringPtr("API configuration updated successfully"), - Id: stringPtr(existing.GetHandle()), - UpdatedAt: timePtr(existing.UpdatedAt), + Id: stringPtr(result.StoredConfig.GetHandle()), + UpdatedAt: timePtr(result.StoredConfig.UpdatedAt), }) - - // Rebuild and update derived policy configuration - if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(existing) - if storedPolicy != nil { - if err := s.policyManager.AddPolicy(storedPolicy); err != nil { - log.Error("Failed to update derived policy configuration", zap.Error(err)) - } else { - log.Info("Derived policy configuration updated", - zap.String("policy_id", storedPolicy.ID), - zap.Int("route_count", len(storedPolicy.Configuration.Routes))) - } - } else { - // API no longer has policies, remove the existing policy configuration - policyID := existing.ID + "-policies" - if err := s.policyManager.RemovePolicy(policyID); err != nil { - // Log at debug level since policy may not exist if API never had policies - log.Debug("No policy configuration to remove", zap.String("policy_id", policyID)) - } else { - log.Info("Derived policy configuration removed (API no longer has policies)", - zap.String("policy_id", policyID)) - } - } - } + // Policy management is now handled by the deployment service } // DeleteAPI implements ServerInterface.DeleteAPI @@ -1175,7 +982,7 @@ func (s *APIServer) CreateLLMProvider(c *gin.Context) { // Build and add policy config derived from API configuration if policies are present if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(stored) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(stored) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to add derived policy configuration", zap.Error(err)) @@ -1265,7 +1072,7 @@ func (s *APIServer) UpdateLLMProvider(c *gin.Context, id string) { // Rebuild and update derived policy configuration if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(updated) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(updated) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to update derived policy configuration", zap.Error(err)) @@ -1412,7 +1219,7 @@ func (s *APIServer) CreateLLMProxy(c *gin.Context) { // Build and add policy config derived from API configuration if policies are present if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(stored) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(stored) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to add derived policy configuration", zap.Error(err)) @@ -1502,7 +1309,7 @@ func (s *APIServer) UpdateLLMProxy(c *gin.Context, id string) { // Rebuild and update derived policy configuration if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(updated) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(updated) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to update derived policy configuration", zap.Error(err)) @@ -1593,182 +1400,6 @@ func (s *APIServer) ListPolicies(c *gin.Context) { c.JSON(http.StatusOK, resp) } -// buildStoredPolicyFromAPI constructs a StoredPolicyConfig from an API config -// Merging rules: When operation has policies, they define the order (can reorder, override, or extend API policies). -// Remaining API-level policies not mentioned in operation policies are appended at the end. -// When operation has no policies, API-level policies are used in their declared order. -// RouteKey uses the fully qualified route path (context + operation path) and must match the route name format -// used by the xDS translator for consistency. -func (s *APIServer) buildStoredPolicyFromAPI(cfg *models.StoredConfig) *models.StoredPolicyConfig { - // TODO: (renuka) duplicate buildStoredPolicyFromAPI funcs. Refactor this. - apiCfg := &cfg.Configuration - - // Collect API-level policies - apiPolicies := make(map[string]policyenginev1.PolicyInstance) // name -> policy - if cfg.GetPolicies() != nil { - for _, p := range *cfg.GetPolicies() { - apiPolicies[p.Name] = convertAPIPolicy(p) - } - } - - routes := make([]policyenginev1.PolicyChain, 0) - switch apiCfg.Kind { - case api.Asyncwebsub: - // Build routes with merged policies - apiData, err := apiCfg.Spec.AsWebhookAPIData() - if err != nil { - // Handle error appropriately (e.g., log or return) - return nil - } - for _, ch := range apiData.Channels { - var finalPolicies []policyenginev1.PolicyInstance - - if ch.Policies != nil && len(*ch.Policies) > 0 { - // Operation has policies: use operation policy order as authoritative - // This allows operations to reorder, override, or extend API-level policies - finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*ch.Policies)) - addedNames := make(map[string]struct{}) - - for _, opPolicy := range *ch.Policies { - finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) - addedNames[opPolicy.Name] = struct{}{} - } - - // Add any API-level policies not mentioned in operation policies (append at end) - if apiData.Policies != nil { - for _, apiPolicy := range *apiData.Policies { - if _, exists := addedNames[apiPolicy.Name]; !exists { - finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) - } - } - } - } else { - // No operation policies: use API-level policies in their declared order - if apiData.Policies != nil { - finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) - for _, p := range *apiData.Policies { - finalPolicies = append(finalPolicies, apiPolicies[p.Name]) - } - } - } - - routeKey := xds.GenerateRouteName("POST", apiData.Context, apiData.Version, ch.Path, s.routerConfig.GatewayHost) - routes = append(routes, policyenginev1.PolicyChain{ - RouteKey: routeKey, - Policies: finalPolicies, - }) - } - case api.RestApi: - // Build routes with merged policies - apiData, err := apiCfg.Spec.AsAPIConfigData() - if err != nil { - // Handle error appropriately (e.g., log or return) - return nil - } - for _, op := range apiData.Operations { - var finalPolicies []policyenginev1.PolicyInstance - - if op.Policies != nil && len(*op.Policies) > 0 { - // Operation has policies: use operation policy order as authoritative - // This allows operations to reorder, override, or extend API-level policies - finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*op.Policies)) - addedNames := make(map[string]struct{}) - - for _, opPolicy := range *op.Policies { - finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) - addedNames[opPolicy.Name] = struct{}{} - } - - // Add any API-level policies not mentioned in operation policies (append at end) - if apiData.Policies != nil { - for _, apiPolicy := range *apiData.Policies { - if _, exists := addedNames[apiPolicy.Name]; !exists { - finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) - } - } - } - } else { - // No operation policies: use API-level policies in their declared order - if apiData.Policies != nil { - finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) - for _, p := range *apiData.Policies { - finalPolicies = append(finalPolicies, apiPolicies[p.Name]) - } - } - } - - // Determine effective vhosts (fallback to global router defaults when not provided) - effectiveMainVHost := s.routerConfig.VHosts.Main.Default - effectiveSandboxVHost := s.routerConfig.VHosts.Sandbox.Default - if apiData.Vhosts != nil { - if strings.TrimSpace(apiData.Vhosts.Main) != "" { - effectiveMainVHost = apiData.Vhosts.Main - } - if apiData.Vhosts.Sandbox != nil && strings.TrimSpace(*apiData.Vhosts.Sandbox) != "" { - effectiveSandboxVHost = *apiData.Vhosts.Sandbox - } - } - - vhosts := []string{effectiveMainVHost} - if apiData.Upstream.Sandbox != nil && apiData.Upstream.Sandbox.Url != nil && - strings.TrimSpace(*apiData.Upstream.Sandbox.Url) != "" { - vhosts = append(vhosts, effectiveSandboxVHost) - } - - for _, vhost := range vhosts { - routes = append(routes, policyenginev1.PolicyChain{ - RouteKey: xds.GenerateRouteName(string(op.Method), apiData.Context, apiData.Version, op.Path, vhost), - Policies: finalPolicies, - }) - } - } - } - - // If there are no policies at all, return nil (skip creation) - policyCount := 0 - for _, r := range routes { - policyCount += len(r.Policies) - } - if policyCount == 0 { - return nil - } - - now := time.Now().Unix() - stored := &models.StoredPolicyConfig{ - ID: cfg.ID + "-policies", - Configuration: policyenginev1.Configuration{ - Routes: routes, - Metadata: policyenginev1.Metadata{ - CreatedAt: now, - UpdatedAt: now, - ResourceVersion: 0, - APIName: cfg.GetDisplayName(), - Version: cfg.GetVersion(), - Context: cfg.GetContext(), - }, - }, - Version: 0, - } - return stored -} - -// convertAPIPolicy converts generated api.Policy to policyenginev1.PolicyInstance -func convertAPIPolicy(p api.Policy) policyenginev1.PolicyInstance { - paramsMap := make(map[string]interface{}) - if p.Params != nil { - for k, v := range *p.Params { - paramsMap[k] = v - } - } - return policyenginev1.PolicyInstance{ - Name: p.Name, - Version: p.Version, - Enabled: true, // Default to enabled - ExecutionCondition: p.ExecutionCondition, - Parameters: paramsMap, - } -} - // CreateMCPProxy implements ServerInterface.CreateMCPProxy // (POST /mcp-proxies) func (s *APIServer) CreateMCPProxy(c *gin.Context) { @@ -1830,7 +1461,7 @@ func (s *APIServer) CreateMCPProxy(c *gin.Context) { // Build and add policy config derived from API configuration if policies are present if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(cfg) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(cfg) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to add derived policy configuration", zap.Error(err)) @@ -2007,7 +1638,7 @@ func (s *APIServer) UpdateMCPProxy(c *gin.Context, id string) { // Rebuild and update derived policy configuration if s.policyManager != nil { - storedPolicy := s.buildStoredPolicyFromAPI(updated) + storedPolicy := s.deploymentService.BuildStoredPolicyFromAPI(updated) if storedPolicy != nil { if err := s.policyManager.AddPolicy(storedPolicy); err != nil { log.Error("Failed to update derived policy configuration", zap.Error(err)) diff --git a/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go b/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go index 1009978a6..b0b7ca723 100644 --- a/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go +++ b/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go @@ -26,6 +26,8 @@ import ( api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/utils" ) // newTestAPIServer creates a minimal APIServer instance for testing @@ -34,11 +36,15 @@ func newTestAPIServer() *APIServer { Main: config.VHostEntry{Default: "localhost"}, Sandbox: config.VHostEntry{Default: "sandbox-*"}, } + routerConfig := &config.RouterConfig{ + GatewayHost: "localhost", + VHosts: *vhosts, + } + configStore := storage.NewConfigStore() + deploymentService := utils.NewAPIDeploymentService(configStore, nil, nil, nil, nil, routerConfig) return &APIServer{ - routerConfig: &config.RouterConfig{ - GatewayHost: "localhost", - VHosts: *vhosts, - }, + routerConfig: routerConfig, + deploymentService: deploymentService, } } @@ -194,7 +200,7 @@ func TestPolicyOrderingDeterministic(t *testing.T) { // Call the function server := newTestAPIServer() - result := server.buildStoredPolicyFromAPI(cfg) // Verify result is not nil when policies exist + result := server.deploymentService.BuildStoredPolicyFromAPI(cfg) // Verify result is not nil when policies exist if len(tt.expectedOrder) > 0 { require.NotNil(t, result, tt.description) require.Len(t, result.Configuration.Routes, 1, "Should have one route") @@ -308,7 +314,7 @@ func TestMultipleOperationsIndependentPolicies(t *testing.T) { } server := newTestAPIServer() - result := server.buildStoredPolicyFromAPI(cfg) + result := server.deploymentService.BuildStoredPolicyFromAPI(cfg) require.NotNil(t, result) require.Len(t, result.Configuration.Routes, 5, "Should have 5 routes") @@ -435,7 +441,7 @@ func TestPolicyOrderingConsistency(t *testing.T) { var firstOrder []string server := newTestAPIServer() for i := 0; i < 100; i++ { - result := server.buildStoredPolicyFromAPI(cfg) + result := server.deploymentService.BuildStoredPolicyFromAPI(cfg) require.NotNil(t, result) require.Len(t, result.Configuration.Routes, 1) diff --git a/gateway/gateway-controller/pkg/controlplane/client.go b/gateway/gateway-controller/pkg/controlplane/client.go index ad82be485..4b7822084 100644 --- a/gateway/gateway-controller/pkg/controlplane/client.go +++ b/gateway/gateway-controller/pkg/controlplane/client.go @@ -32,6 +32,7 @@ import ( "github.com/gorilla/websocket" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/utils" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" @@ -113,6 +114,7 @@ func NewClient( store *storage.ConfigStore, db storage.Storage, snapshotManager *xds.SnapshotManager, + policyManager *policyxds.PolicyManager, validator config.Validator, routerConfig *config.RouterConfig, ) *Client { @@ -126,7 +128,7 @@ func NewClient( snapshotManager: snapshotManager, parser: config.NewParser(), validator: validator, - deploymentService: utils.NewAPIDeploymentService(store, db, snapshotManager, validator, routerConfig), + deploymentService: utils.NewAPIDeploymentService(store, db, snapshotManager, policyManager, validator, routerConfig), state: &ConnectionState{ Current: Disconnected, Conn: nil, diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 46af53f27..5fd0743e3 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -30,8 +30,10 @@ import ( "github.com/google/uuid" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" + policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" "go.uber.org/zap" @@ -52,11 +54,85 @@ type APIDeploymentResult struct { IsUpdate bool } +// APIUpdateParams contains parameters for API update operations +type APIUpdateParams struct { + Handle string // API handle from URL path + Data []byte // Raw configuration data (YAML/JSON) + ContentType string // Content type for parsing + CorrelationID string // Correlation ID for tracking + Logger *zap.Logger // Logger instance +} + +// Custom error types for HTTP status code mapping + +// NotFoundError indicates the requested resource was not found +type NotFoundError struct { + Handle string +} + +func (e *NotFoundError) Error() string { + return fmt.Sprintf("API configuration with handle '%s' not found", e.Handle) +} + +// HandleMismatchError indicates the handle in the URL doesn't match the YAML metadata +type HandleMismatchError struct { + PathHandle string + YamlHandle string +} + +func (e *HandleMismatchError) Error() string { + return fmt.Sprintf("Handle mismatch: path has '%s' but YAML metadata.name has '%s'", e.PathHandle, e.YamlHandle) +} + +// APIValidationError wraps validation errors for API configurations +type APIValidationError struct { + Errors []config.ValidationError +} + +func (e *APIValidationError) Error() string { + return fmt.Sprintf("configuration validation failed with %d errors", len(e.Errors)) +} + +// TopicOperationError indicates WebSub topic operations failed +type TopicOperationError struct { + Message string +} + +func (e *TopicOperationError) Error() string { + return e.Message +} + +// ConflictError indicates a resource conflict (e.g., handle already exists) +type ConflictError struct { + Message string +} + +func (e *ConflictError) Error() string { + return e.Message +} + +// ParseError indicates configuration parsing failed +type ParseError struct { + Message string +} + +func (e *ParseError) Error() string { + return e.Message +} + +// DatabaseUnavailableError indicates the database is not available +type DatabaseUnavailableError struct{} + +func (e *DatabaseUnavailableError) Error() string { + return "Database storage not available" +} + // APIDeploymentService provides utilities for API configuration deployment type APIDeploymentService struct { store *storage.ConfigStore db storage.Storage snapshotManager *xds.SnapshotManager + policyManager *policyxds.PolicyManager parser *config.Parser validator config.Validator routerConfig *config.RouterConfig @@ -68,6 +144,7 @@ func NewAPIDeploymentService( store *storage.ConfigStore, db storage.Storage, snapshotManager *xds.SnapshotManager, + policyManager *policyxds.PolicyManager, validator config.Validator, routerConfig *config.RouterConfig, ) *APIDeploymentService { @@ -75,6 +152,7 @@ func NewAPIDeploymentService( store: store, db: db, snapshotManager: snapshotManager, + policyManager: policyManager, parser: config.NewParser(), validator: validator, httpClient: &http.Client{Timeout: 10 * time.Second}, @@ -164,86 +242,10 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams DeployedVersion: 0, } + // Handle AsyncWebSub topic lifecycle management if apiConfig.Kind == api.Asyncwebsub { - topicsToRegister, topicsToUnregister := s.GetTopicsForUpdate(*storedCfg) - // TODO: Pre configure the dynamic forward proxy rules for event gw - // This was communication bridge will be created on the gw startup - // Can perform internal communication with websub hub without relying on the dynamic rules - // Execute topic operations with wait group and errors tracking - var wg2 sync.WaitGroup - var regErrs int32 - var deregErrs int32 - - if len(topicsToRegister) > 0 { - wg2.Add(1) - go func(list []string) { - defer wg2.Done() - params.Logger.Info("Starting topic registration", zap.Int("total_topics", len(list)), zap.String("api_id", apiID)) - var childWg sync.WaitGroup - for _, topic := range list { - childWg.Add(1) - go func(topic string) { - defer childWg.Done() - if err := s.RegisterTopicWithHub(s.httpClient, topic, "localhost", 8083, params.Logger); err != nil { - params.Logger.Error("Failed to register topic with WebSubHub", - zap.Error(err), - zap.String("topic", topic), - zap.String("api_id", apiID)) - atomic.AddInt32(®Errs, 1) - return - } else { - params.Logger.Info("Successfully registered topic with WebSubHub", - zap.String("topic", topic), - zap.String("api_id", apiID)) - } - }(topic) - } - childWg.Wait() - }(topicsToRegister) - } - - if len(topicsToUnregister) > 0 { - wg2.Add(1) - go func(list []string) { - defer wg2.Done() - var childWg sync.WaitGroup - params.Logger.Info("Starting topic deregistration", zap.Int("total_topics", len(list)), zap.String("api_id", apiID)) - for _, topic := range list { - childWg.Add(1) - go func(topic string) { - defer childWg.Done() - if err := s.UnregisterTopicWithHub(s.httpClient, topic, "localhost", 8083, params.Logger); err != nil { - params.Logger.Error("Failed to deregister topic from WebSubHub", - zap.Error(err), - zap.String("topic", topic), - zap.String("api_id", apiID)) - atomic.AddInt32(&deregErrs, 1) - return - } else { - params.Logger.Info("Successfully deregistered topic from WebSubHub", - zap.String("topic", topic), - zap.String("api_id", apiID)) - } - }(topic) - } - childWg.Wait() - }(topicsToUnregister) - } - - wg2.Wait() - params.Logger.Info("Topic lifecycle operations completed", - zap.String("api_id", apiID), - zap.Int("registered", len(topicsToRegister)), - zap.Int("deregistered", len(topicsToUnregister)), - zap.Int("register_errors", int(regErrs)), - zap.Int("deregister_errors", int(deregErrs))) - - // Check if topic operations failed and return error - if regErrs > 0 || deregErrs > 0 { - params.Logger.Error("Topic lifecycle operations failed", - zap.Int("register_errors", int(regErrs)), - zap.Int("deregister_errors", int(deregErrs))) - return nil, fmt.Errorf("failed to complete topic operations: %d registration error(s), %d deregistration error(s)", regErrs, deregErrs) + if err := s.handleTopicLifecycle(storedCfg, params.Logger); err != nil { + return nil, err } } @@ -269,17 +271,10 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams } // Update xDS snapshot asynchronously - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + s.triggerXDSSnapshotUpdate(apiID, params.CorrelationID, params.Logger) - if err := s.snapshotManager.UpdateSnapshot(ctx, params.CorrelationID); err != nil { - params.Logger.Error("Failed to update xDS snapshot", - zap.Error(err), - zap.String("api_id", apiID), - zap.String("correlation_id", params.CorrelationID)) - } - }() + // Update derived policy configuration + s.updatePolicyConfiguration(storedCfg, params.Logger) return &APIDeploymentResult{ StoredConfig: storedCfg, @@ -287,6 +282,188 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams }, nil } +// UpdateAPIConfiguration handles the complete API configuration update process +func (s *APIDeploymentService) UpdateAPIConfiguration(params APIUpdateParams) (*APIDeploymentResult, error) { + handle := params.Handle + + // Parse configuration + var apiConfig api.APIConfiguration + err := s.parser.Parse(params.Data, params.ContentType, &apiConfig) + if err != nil { + params.Logger.Error("Failed to parse configuration", zap.Error(err)) + return nil, &ParseError{Message: "Failed to parse configuration"} + } + + // Validate that the handle in the YAML matches the path parameter + if apiConfig.Metadata.Name != "" { + if apiConfig.Metadata.Name != handle { + params.Logger.Warn("Handle mismatch between path and YAML metadata", + zap.String("path_handle", handle), + zap.String("yaml_handle", apiConfig.Metadata.Name)) + return nil, &HandleMismatchError{ + PathHandle: handle, + YamlHandle: apiConfig.Metadata.Name, + } + } + } + + // Validate configuration + validationErrors := s.validator.Validate(&apiConfig) + if len(validationErrors) > 0 { + params.Logger.Warn("Configuration validation failed", + zap.String("handle", handle), + zap.Int("num_errors", len(validationErrors))) + return nil, &APIValidationError{Errors: validationErrors} + } + + if s.db == nil { + return nil, &DatabaseUnavailableError{} + } + + // Check if config exists + existing, err := s.db.GetConfigByHandle(handle) + if err != nil { + params.Logger.Warn("API configuration not found", zap.String("handle", handle)) + return nil, &NotFoundError{Handle: handle} + } + + // Update stored configuration + now := time.Now() + existing.Configuration = apiConfig + existing.Status = models.StatusPending + existing.UpdatedAt = now + existing.DeployedAt = nil + existing.DeployedVersion = 0 + + // Handle AsyncWebSub topic lifecycle management + if apiConfig.Kind == api.Asyncwebsub { + if err := s.handleTopicLifecycle(existing, params.Logger); err != nil { + return nil, err + } + } + + // Update database first (only if persistent mode) + if s.db != nil { + if err := s.db.UpdateConfig(existing); err != nil { + params.Logger.Error("Failed to update config in database", zap.Error(err)) + return nil, fmt.Errorf("failed to persist configuration update: %w", err) + } + } + + // Update in-memory store + if err := s.store.Update(existing); err != nil { + if storage.IsConflictError(err) { + params.Logger.Info("API configuration handle already exists", + zap.String("id", existing.ID), + zap.String("handle", handle)) + return nil, &ConflictError{Message: err.Error()} + } + params.Logger.Error("Failed to update config in memory store", zap.Error(err)) + return nil, fmt.Errorf("failed to update configuration in memory store: %w", err) + } + + params.Logger.Info("API configuration updated", + zap.String("id", existing.ID), + zap.String("handle", handle)) + + // Update xDS snapshot asynchronously + s.triggerXDSSnapshotUpdate(existing.ID, params.CorrelationID, params.Logger) + + // Update derived policy configuration + s.updatePolicyConfiguration(existing, params.Logger) + + return &APIDeploymentResult{ + StoredConfig: existing, + IsUpdate: true, + }, nil +} + +// handleTopicLifecycle manages WebSub topic registration/deregistration for AsyncWebSub APIs +func (s *APIDeploymentService) handleTopicLifecycle(storedCfg *models.StoredConfig, logger *zap.Logger) error { + topicsToRegister, topicsToUnregister := s.GetTopicsForUpdate(*storedCfg) + + var wg sync.WaitGroup + var regErrs int32 + var deregErrs int32 + + // Register new topics + if len(topicsToRegister) > 0 { + wg.Add(1) + go func(list []string) { + defer wg.Done() + logger.Info("Starting topic registration", + zap.Int("total_topics", len(list)), + zap.String("api_id", storedCfg.ID)) + var childWg sync.WaitGroup + for _, topic := range list { + childWg.Add(1) + go func(topic string) { + defer childWg.Done() + if err := s.RegisterTopicWithHub(s.httpClient, topic, "localhost", 8083, logger); err != nil { + logger.Error("Failed to register topic with WebSubHub", + zap.Error(err), + zap.String("topic", topic), + zap.String("api_id", storedCfg.ID)) + atomic.AddInt32(®Errs, 1) + } else { + logger.Info("Successfully registered topic with WebSubHub", + zap.String("topic", topic), + zap.String("api_id", storedCfg.ID)) + } + }(topic) + } + childWg.Wait() + }(topicsToRegister) + } + + // Deregister removed topics + if len(topicsToUnregister) > 0 { + wg.Add(1) + go func(list []string) { + defer wg.Done() + logger.Info("Starting topic deregistration", + zap.Int("total_topics", len(list)), + zap.String("api_id", storedCfg.ID)) + var childWg sync.WaitGroup + for _, topic := range list { + childWg.Add(1) + go func(topic string) { + defer childWg.Done() + if err := s.UnregisterTopicWithHub(s.httpClient, topic, "localhost", 8083, logger); err != nil { + logger.Error("Failed to deregister topic from WebSubHub", + zap.Error(err), + zap.String("topic", topic), + zap.String("api_id", storedCfg.ID)) + atomic.AddInt32(&deregErrs, 1) + } else { + logger.Info("Successfully deregistered topic from WebSubHub", + zap.String("topic", topic), + zap.String("api_id", storedCfg.ID)) + } + }(topic) + } + childWg.Wait() + }(topicsToUnregister) + } + + wg.Wait() + + logger.Info("Topic lifecycle operations completed", + zap.String("api_id", storedCfg.ID), + zap.Int("registered", len(topicsToRegister)), + zap.Int("deregistered", len(topicsToUnregister)), + zap.Int("register_errors", int(regErrs)), + zap.Int("deregister_errors", int(deregErrs))) + + if regErrs > 0 || deregErrs > 0 { + return &TopicOperationError{ + Message: fmt.Sprintf("Topic lifecycle operations failed: %d registration error(s), %d deregistration error(s)", regErrs, deregErrs), + } + } + + return nil +} + func (s *APIDeploymentService) GetTopicsForUpdate(apiConfig models.StoredConfig) ([]string, []string) { topics := s.store.TopicManager.GetAllByConfig(apiConfig.ID) topicsToRegister := []string{} @@ -496,3 +673,215 @@ func (s *APIDeploymentService) sendTopicRequestToHub(httpClient *http.Client, to func generateUUID() string { return uuid.New().String() } + +// BuildStoredPolicyFromAPI builds a StoredPolicyConfig from an API configuration. +// This builds policy chains for each route based on API-level and operation-level policies. +// RouteKey uses the fully qualified route path (context + operation path) and must match +// the route name format used by the xDS translator for consistency. +func (s *APIDeploymentService) BuildStoredPolicyFromAPI(cfg *models.StoredConfig) *models.StoredPolicyConfig { + apiCfg := &cfg.Configuration + + // Collect API-level policies + apiPolicies := make(map[string]policyenginev1.PolicyInstance) // name -> policy + if cfg.GetPolicies() != nil { + for _, p := range *cfg.GetPolicies() { + apiPolicies[p.Name] = convertAPIPolicy(p) + } + } + + routes := make([]policyenginev1.PolicyChain, 0) + switch apiCfg.Kind { + case api.Asyncwebsub: + // Build routes with merged policies + apiData, err := apiCfg.Spec.AsWebhookAPIData() + if err != nil { + return nil + } + for _, ch := range apiData.Channels { + var finalPolicies []policyenginev1.PolicyInstance + + if ch.Policies != nil && len(*ch.Policies) > 0 { + // Operation has policies: use operation policy order as authoritative + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*ch.Policies)) + addedNames := make(map[string]struct{}) + + for _, opPolicy := range *ch.Policies { + finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) + addedNames[opPolicy.Name] = struct{}{} + } + + // Add any API-level policies not mentioned in operation policies (append at end) + if apiData.Policies != nil { + for _, apiPolicy := range *apiData.Policies { + if _, exists := addedNames[apiPolicy.Name]; !exists { + finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) + } + } + } + } else { + // No operation policies: use API-level policies in their declared order + if apiData.Policies != nil { + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) + for _, p := range *apiData.Policies { + finalPolicies = append(finalPolicies, apiPolicies[p.Name]) + } + } + } + + routeKey := xds.GenerateRouteName("POST", apiData.Context, apiData.Version, ch.Path, s.routerConfig.GatewayHost) + routes = append(routes, policyenginev1.PolicyChain{ + RouteKey: routeKey, + Policies: finalPolicies, + }) + } + case api.RestApi: + // Build routes with merged policies + apiData, err := apiCfg.Spec.AsAPIConfigData() + if err != nil { + return nil + } + for _, op := range apiData.Operations { + var finalPolicies []policyenginev1.PolicyInstance + + if op.Policies != nil && len(*op.Policies) > 0 { + // Operation has policies: use operation policy order as authoritative + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*op.Policies)) + addedNames := make(map[string]struct{}) + + for _, opPolicy := range *op.Policies { + finalPolicies = append(finalPolicies, convertAPIPolicy(opPolicy)) + addedNames[opPolicy.Name] = struct{}{} + } + + // Add any API-level policies not mentioned in operation policies (append at end) + if apiData.Policies != nil { + for _, apiPolicy := range *apiData.Policies { + if _, exists := addedNames[apiPolicy.Name]; !exists { + finalPolicies = append(finalPolicies, apiPolicies[apiPolicy.Name]) + } + } + } + } else { + // No operation policies: use API-level policies in their declared order + if apiData.Policies != nil { + finalPolicies = make([]policyenginev1.PolicyInstance, 0, len(*apiData.Policies)) + for _, p := range *apiData.Policies { + finalPolicies = append(finalPolicies, apiPolicies[p.Name]) + } + } + } + + // Determine effective vhosts (fallback to global router defaults when not provided) + effectiveMainVHost := s.routerConfig.VHosts.Main.Default + effectiveSandboxVHost := s.routerConfig.VHosts.Sandbox.Default + if apiData.Vhosts != nil { + if strings.TrimSpace(apiData.Vhosts.Main) != "" { + effectiveMainVHost = apiData.Vhosts.Main + } + if apiData.Vhosts.Sandbox != nil && strings.TrimSpace(*apiData.Vhosts.Sandbox) != "" { + effectiveSandboxVHost = *apiData.Vhosts.Sandbox + } + } + + vhosts := []string{effectiveMainVHost} + if apiData.Upstream.Sandbox != nil && apiData.Upstream.Sandbox.Url != nil && + strings.TrimSpace(*apiData.Upstream.Sandbox.Url) != "" { + vhosts = append(vhosts, effectiveSandboxVHost) + } + + for _, vhost := range vhosts { + routes = append(routes, policyenginev1.PolicyChain{ + RouteKey: xds.GenerateRouteName(string(op.Method), apiData.Context, apiData.Version, op.Path, vhost), + Policies: finalPolicies, + }) + } + } + } + + // If there are no policies at all, return nil (skip creation) + policyCount := 0 + for _, r := range routes { + policyCount += len(r.Policies) + } + if policyCount == 0 { + return nil + } + + now := time.Now().Unix() + stored := &models.StoredPolicyConfig{ + ID: cfg.ID + "-policies", + Configuration: policyenginev1.Configuration{ + Routes: routes, + Metadata: policyenginev1.Metadata{ + CreatedAt: now, + UpdatedAt: now, + ResourceVersion: 0, + APIName: cfg.GetDisplayName(), + Version: cfg.GetVersion(), + Context: cfg.GetContext(), + }, + }, + Version: 0, + } + return stored +} + +// convertAPIPolicy converts generated api.Policy to policyenginev1.PolicyInstance +func convertAPIPolicy(p api.Policy) policyenginev1.PolicyInstance { + paramsMap := make(map[string]interface{}) + if p.Params != nil { + for k, v := range *p.Params { + paramsMap[k] = v + } + } + return policyenginev1.PolicyInstance{ + Name: p.Name, + Version: p.Version, + Enabled: true, // Default to enabled + ExecutionCondition: p.ExecutionCondition, + Parameters: paramsMap, + } +} + +// updatePolicyConfiguration builds and updates/removes derived policy config for an API +func (s *APIDeploymentService) updatePolicyConfiguration(storedCfg *models.StoredConfig, logger *zap.Logger) { + if s.policyManager == nil { + return + } + + storedPolicy := s.BuildStoredPolicyFromAPI(storedCfg) + if storedPolicy != nil { + if err := s.policyManager.AddPolicy(storedPolicy); err != nil { + logger.Error("Failed to update derived policy configuration", zap.Error(err)) + } else { + logger.Info("Derived policy configuration updated", + zap.String("policy_id", storedPolicy.ID), + zap.Int("route_count", len(storedPolicy.Configuration.Routes))) + } + } else { + // API no longer has policies, remove the existing policy configuration + policyID := storedCfg.ID + "-policies" + if err := s.policyManager.RemovePolicy(policyID); err != nil { + // Log at debug level since policy may not exist if API never had policies + logger.Debug("No policy configuration to remove", zap.String("policy_id", policyID)) + } else { + logger.Info("Derived policy configuration removed (API no longer has policies)", + zap.String("policy_id", policyID)) + } + } +} + +// triggerXDSSnapshotUpdate asynchronously updates xDS snapshot +func (s *APIDeploymentService) triggerXDSSnapshotUpdate(apiID, correlationID string, logger *zap.Logger) { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := s.snapshotManager.UpdateSnapshot(ctx, correlationID); err != nil { + logger.Error("Failed to update xDS snapshot", + zap.Error(err), + zap.String("api_id", apiID), + zap.String("correlation_id", correlationID)) + } + }() +} diff --git a/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go b/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go index 2297d9fa1..d96d4f2c3 100644 --- a/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go +++ b/gateway/gateway-controller/pkg/utils/websub_topic_registration_test.go @@ -18,7 +18,7 @@ func TestDeployAPIConfigurationWebSubKindTopicRegistration(t *testing.T) { db := &storage.SQLiteStorage{} snapshotManager := &xds.SnapshotManager{} validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, db, snapshotManager, validator, nil) + service := NewAPIDeploymentService(configStore, db, snapshotManager, nil, validator, nil) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub @@ -63,7 +63,7 @@ spec: func TestDeployAPIConfigurationWebSubKindRevisionDeployment(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, nil, validator, nil) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub @@ -145,7 +145,7 @@ spec: func TestTopicRegistrationForConcurrentAPIConfigs(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, nil, validator, nil) // Two different API YAMLs yamlA := `kind: async/websub @@ -249,7 +249,7 @@ spec: func TestTopicDeregistrationOnConfigDeletion(t *testing.T) { configStore := storage.NewConfigStore() validator := config.NewAPIValidator() - service := NewAPIDeploymentService(configStore, nil, nil, validator, nil) + service := NewAPIDeploymentService(configStore, nil, nil, nil, validator, nil) // Inline YAML config similar to websubhub.yaml yamlConfig := `kind: async/websub From 3ae57d6f5c20878839899d1a9d152c0f7e9df16e Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Tue, 6 Jan 2026 14:26:52 +0530 Subject: [PATCH 17/24] Remove todo --- gateway/gateway-controller/pkg/api/handlers/handlers.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/gateway/gateway-controller/pkg/api/handlers/handlers.go b/gateway/gateway-controller/pkg/api/handlers/handlers.go index 309323167..6c0d94b2d 100644 --- a/gateway/gateway-controller/pkg/api/handlers/handlers.go +++ b/gateway/gateway-controller/pkg/api/handlers/handlers.go @@ -516,7 +516,6 @@ func (s *APIServer) UpdateAPI(c *gin.Context, id string) { return } - // TODO: (VirajSalaka) Needs to implement this based on deploymentService DeployAPIConfiguration // Validate that the handle in the YAML matches the path parameter if apiConfig.Metadata.Name != "" { if apiConfig.Metadata.Name != handle { @@ -703,7 +702,6 @@ func (s *APIServer) UpdateAPI(c *gin.Context, id string) { correlationID := middleware.GetCorrelationID(c) // Update xDS snapshot asynchronously - // TODO: (VirajSalaka) Fix to work with eventhub go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() From 95bddd7bf22b3501951e7dee75f25abd96911ee1 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 7 Jan 2026 11:55:03 +0530 Subject: [PATCH 18/24] Fix test failures --- .../gateway-controller/pkg/api/handlers/policy_ordering_test.go | 2 +- gateway/gateway-controller/tests/integration/schema_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go b/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go index b0b7ca723..3b706e5ca 100644 --- a/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go +++ b/gateway/gateway-controller/pkg/api/handlers/policy_ordering_test.go @@ -41,7 +41,7 @@ func newTestAPIServer() *APIServer { VHosts: *vhosts, } configStore := storage.NewConfigStore() - deploymentService := utils.NewAPIDeploymentService(configStore, nil, nil, nil, nil, routerConfig) + deploymentService := utils.NewAPIDeploymentService(configStore, nil, nil, nil, nil, routerConfig, nil, false) return &APIServer{ routerConfig: routerConfig, deploymentService: deploymentService, diff --git a/gateway/gateway-controller/tests/integration/schema_test.go b/gateway/gateway-controller/tests/integration/schema_test.go index f964e61d1..b7f73f5d7 100644 --- a/gateway/gateway-controller/tests/integration/schema_test.go +++ b/gateway/gateway-controller/tests/integration/schema_test.go @@ -95,7 +95,7 @@ func TestSchemaInitialization(t *testing.T) { var version int err := rawDB.QueryRow("PRAGMA user_version").Scan(&version) assert.NoError(t, err) - assert.Equal(t, 5, version, "Schema version should be 5") + assert.Equal(t, 9, version, "Schema version should be 5") }) // Verify deployments table exists From 8fa1f340c2487cdaede44911e9ea13af79da5577 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 7 Jan 2026 13:02:06 +0530 Subject: [PATCH 19/24] Refactor --- gateway/gateway-controller/cmd/controller/main.go | 6 +++--- gateway/gateway-controller/pkg/api/handlers/handlers.go | 4 ++-- gateway/gateway-controller/pkg/utils/api_deployment.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 9bfad35af..19bcead6c 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -110,11 +110,11 @@ func main() { // Initialize in-memory API key store for xDS apiKeyStore := storage.NewAPIKeyStore(log) - // Initialize EventHub if multi-tenant mode is enabled + // Initialize EventHub if multi-replica mode is enabled var eventHub eventhub.EventHub if cfg.GatewayController.Server.EnableReplicaSync { if cfg.IsPersistentMode() && db != nil { - log.Info("Initializing EventHub for multi-tenant mode") + log.Info("Initializing EventHub for multi-replica mode") eventHub = eventhub.New(db.GetDB(), log, eventhub.DefaultConfig()) ctx := context.Background() if err := eventHub.Initialize(ctx); err != nil { @@ -123,7 +123,7 @@ func main() { eventHub.RegisterOrganization("default") log.Info("EventHub initialized successfully") } else { - log.Fatal("EventHub requires persistent storage. Multi-tenant mode will not function correctly in memory-only mode.") + log.Fatal("EventHub requires persistent storage. Multi-replica mode will not function correctly in memory-only mode.") } } diff --git a/gateway/gateway-controller/pkg/api/handlers/handlers.go b/gateway/gateway-controller/pkg/api/handlers/handlers.go index e6541ed18..bcb7f8f1f 100644 --- a/gateway/gateway-controller/pkg/api/handlers/handlers.go +++ b/gateway/gateway-controller/pkg/api/handlers/handlers.go @@ -82,9 +82,9 @@ func NewAPIServer( routerConfig *config.RouterConfig, apiKeyXDSManager *apikeyxds.APIKeyStateManager, eventHub eventhub.EventHub, - enableMultiTenantMode bool, + enableMultiReplicaMode bool, ) *APIServer { - deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, policyManager, validator, routerConfig, eventHub, enableMultiTenantMode) + deploymentService := utils.NewAPIDeploymentService(store, db, snapshotManager, policyManager, validator, routerConfig, eventHub, enableMultiReplicaMode) server := &APIServer{ store: store, db: db, diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 5c31b884d..a087a87c8 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -307,7 +307,7 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams zap.String("correlation_id", params.CorrelationID)) } } else { - params.Logger.Warn("Multi-tenant mode enabled but eventhub is not initialized", + params.Logger.Warn("Multi-replica mode enabled but eventhub is not initialized", zap.String("api_id", apiID)) } } else { From 278c8a90636b8bf335fa574cb3a200bb67d10f2e Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 7 Jan 2026 13:35:20 +0530 Subject: [PATCH 20/24] Fix Database Schema migration related changes --- .../pkg/storage/gateway-controller-db.sql | 8 +- .../gateway-controller/pkg/storage/sqlite.go | 139 +----------------- .../tests/integration/schema_test.go | 2 +- 3 files changed, 13 insertions(+), 136 deletions(-) diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index 05d51c047..68663e7d3 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -150,7 +150,7 @@ CREATE INDEX IF NOT EXISTS idx_api_key_status ON api_keys(status); CREATE INDEX IF NOT EXISTS idx_api_key_expiry ON api_keys(expires_at); CREATE INDEX IF NOT EXISTS idx_created_by ON api_keys(created_by); --- EventHub: Organization States Table (added in schema version 9) +-- EventHub: Organization States Table (added in schema version 6) -- Tracks version information per organization for change detection CREATE TABLE IF NOT EXISTS organization_states ( organization TEXT PRIMARY KEY, @@ -160,7 +160,7 @@ CREATE TABLE IF NOT EXISTS organization_states ( CREATE INDEX IF NOT EXISTS idx_organization_states_updated ON organization_states(updated_at); --- EventHub: Unified Events Table (added in schema version 9) +-- EventHub: Unified Events Table (added in schema version 6) -- Stores all entity change events (APIs, certificates, LLM templates, etc.) CREATE TABLE IF NOT EXISTS events ( organization_id TEXT NOT NULL, @@ -177,5 +177,5 @@ CREATE TABLE IF NOT EXISTS events ( CREATE INDEX IF NOT EXISTS idx_events_lookup ON events(organization_id, processed_timestamp); CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type); --- Set schema version to 9 -PRAGMA user_version = 9; +-- Set schema version to 6 +PRAGMA user_version = 6; diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index f70c82914..f28b31276 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -215,120 +215,10 @@ func (s *SQLiteStorage) initSchema() error { } if version == 5 { - // Add event tables for EventHub - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS api_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - organization_id TEXT NOT NULL, - processed_timestamp TIMESTAMP NOT NULL, - originated_timestamp TIMESTAMP NOT NULL, - event_data TEXT NOT NULL - );`); err != nil { - return fmt.Errorf("failed to migrate schema to version 6 (api_events): %w", err) - } - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_api_events_lookup ON api_events(organization_id, processed_timestamp);`); err != nil { - return fmt.Errorf("failed to create api_events index: %w", err) - } - - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS certificate_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - organization_id TEXT NOT NULL, - processed_timestamp TIMESTAMP NOT NULL, - originated_timestamp TIMESTAMP NOT NULL, - event_data TEXT NOT NULL - );`); err != nil { - return fmt.Errorf("failed to migrate schema to version 6 (certificate_events): %w", err) - } - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_cert_events_lookup ON certificate_events(organization_id, processed_timestamp);`); err != nil { - return fmt.Errorf("failed to create certificate_events index: %w", err) - } - - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS llm_template_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - organization_id TEXT NOT NULL, - processed_timestamp TIMESTAMP NOT NULL, - originated_timestamp TIMESTAMP NOT NULL, - event_data TEXT NOT NULL - );`); err != nil { - return fmt.Errorf("failed to migrate schema to version 6 (llm_template_events): %w", err) - } - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_llm_events_lookup ON llm_template_events(organization_id, processed_timestamp);`); err != nil { - return fmt.Errorf("failed to create llm_template_events index: %w", err) - } - - if _, err := s.db.Exec("PRAGMA user_version = 6"); err != nil { - return fmt.Errorf("failed to set schema version to 6: %w", err) - } - s.logger.Info("Schema migrated to version 6 (event tables)") - version = 6 - } - - if version == 6 { - // Add topic_states table - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS topic_states ( - topic_name TEXT PRIMARY KEY, - version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - );`); err != nil { - return fmt.Errorf("failed to migrate schema to version 7 (topic_states): %w", err) - } - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at);`); err != nil { - return fmt.Errorf("failed to create topic_states index: %w", err) - } - if _, err := s.db.Exec("PRAGMA user_version = 7"); err != nil { - return fmt.Errorf("failed to set schema version to 7: %w", err) - } - s.logger.Info("Schema migrated to version 7 (topic_states table)") - version = 7 - } - - if version == 7 { - // Migrate topic_states to include organization as part of primary key - s.logger.Info("Migrating topic_states table to include organization (version 8)") - - // Step 1: Rename old table - if _, err := s.db.Exec(`ALTER TABLE topic_states RENAME TO topic_states_old;`); err != nil { - return fmt.Errorf("failed to rename topic_states table: %w", err) - } - - // Step 2: Create new table with organization - if _, err := s.db.Exec(`CREATE TABLE topic_states ( - organization TEXT NOT NULL, - topic_name TEXT NOT NULL, - version_id TEXT NOT NULL DEFAULT '', - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (organization, topic_name) - );`); err != nil { - return fmt.Errorf("failed to create new topic_states table: %w", err) - } - - // Step 3: Migrate data (set organization to empty string for existing rows) - if _, err := s.db.Exec(`INSERT INTO topic_states (organization, topic_name, version_id, updated_at) - SELECT '', topic_name, version_id, updated_at FROM topic_states_old;`); err != nil { - return fmt.Errorf("failed to migrate topic_states data: %w", err) - } - - // Step 4: Drop old table - if _, err := s.db.Exec(`DROP TABLE topic_states_old;`); err != nil { - return fmt.Errorf("failed to drop old topic_states table: %w", err) - } - - // Step 5: Recreate index - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_topic_states_updated ON topic_states(updated_at);`); err != nil { - return fmt.Errorf("failed to create topic_states index: %w", err) - } - - if _, err := s.db.Exec("PRAGMA user_version = 8"); err != nil { - return fmt.Errorf("failed to set schema version to 8: %w", err) - } - s.logger.Info("Schema migrated to version 8 (topic_states with organization)") - version = 8 - } - - if version == 8 { - // Migrate to organization-centric Event Hub architecture - s.logger.Info("Migrating to organization-centric Event Hub (version 9)") + // Add EventHub tables for multi-replica synchronization + s.logger.Info("Migrating to EventHub schema (version 6)") - // Step 1: Create new organization_states table + // Create organization_states table if _, err := s.db.Exec(`CREATE TABLE organization_states ( organization TEXT PRIMARY KEY, version_id TEXT NOT NULL DEFAULT '', @@ -341,15 +231,7 @@ func (s *SQLiteStorage) initSchema() error { return fmt.Errorf("failed to create organization_states index: %w", err) } - // Step 2: Migrate state data (consolidate per organization) - if _, err := s.db.Exec(`INSERT INTO organization_states (organization, version_id, updated_at) - SELECT organization, MAX(version_id), MAX(updated_at) - FROM topic_states - GROUP BY organization;`); err != nil { - return fmt.Errorf("failed to migrate state data: %w", err) - } - - // Step 3: Create unified events table + // Create unified events table if _, err := s.db.Exec(`CREATE TABLE events ( organization_id TEXT NOT NULL, processed_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -372,17 +254,12 @@ func (s *SQLiteStorage) initSchema() error { return fmt.Errorf("failed to create events type index: %w", err) } - // Step 4: Drop old topic_states table (data already migrated) - if _, err := s.db.Exec(`DROP TABLE topic_states;`); err != nil { - return fmt.Errorf("failed to drop topic_states table: %w", err) - } - - if _, err := s.db.Exec("PRAGMA user_version = 9"); err != nil { - return fmt.Errorf("failed to set schema version to 9: %w", err) + if _, err := s.db.Exec("PRAGMA user_version = 6"); err != nil { + return fmt.Errorf("failed to set schema version to 6: %w", err) } - s.logger.Info("Schema migrated to version 9 (organization-centric Event Hub)") - version = 9 + s.logger.Info("Schema migrated to version 6 (EventHub tables)") + version = 6 } s.logger.Info("Database schema up to date", zap.Int("version", version)) diff --git a/gateway/gateway-controller/tests/integration/schema_test.go b/gateway/gateway-controller/tests/integration/schema_test.go index b7f73f5d7..237505f87 100644 --- a/gateway/gateway-controller/tests/integration/schema_test.go +++ b/gateway/gateway-controller/tests/integration/schema_test.go @@ -95,7 +95,7 @@ func TestSchemaInitialization(t *testing.T) { var version int err := rawDB.QueryRow("PRAGMA user_version").Scan(&version) assert.NoError(t, err) - assert.Equal(t, 9, version, "Schema version should be 5") + assert.Equal(t, 6, version, "Schema version should be 6") }) // Verify deployments table exists From c9f1551f703e37ff0f6e984bf589df42164a468a Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 7 Jan 2026 22:51:36 +0530 Subject: [PATCH 21/24] Resolve commented issues --- .../gateway-controller/cmd/controller/main.go | 4 +- .../pkg/eventhub/eventhub.go | 8 + .../pkg/eventhub/sqlite_backend.go | 26 +- .../pkg/eventlistener/EXAMPLE_USAGE.md | 381 ------------------ .../pkg/eventlistener/api_processor.go | 2 + .../pkg/eventlistener/eventhub_adapter.go | 23 +- .../pkg/storage/gateway-controller-db.sql | 3 - .../gateway-controller/pkg/storage/sqlite.go | 8 - .../pkg/utils/api_deployment.go | 5 +- 9 files changed, 44 insertions(+), 416 deletions(-) delete mode 100644 gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index 19bcead6c..ce6f81b9b 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -120,7 +120,9 @@ func main() { if err := eventHub.Initialize(ctx); err != nil { log.Fatal("Failed to initialize EventHub", zap.Error(err)) } - eventHub.RegisterOrganization("default") + if err := eventHub.RegisterOrganization("default"); err != nil { + log.Fatal("Failed to register default organization", zap.Error(err)) + } log.Info("EventHub initialized successfully") } else { log.Fatal("EventHub requires persistent storage. Multi-replica mode will not function correctly in memory-only mode.") diff --git a/gateway/gateway-controller/pkg/eventhub/eventhub.go b/gateway/gateway-controller/pkg/eventhub/eventhub.go index f31573c78..cf40eabee 100644 --- a/gateway/gateway-controller/pkg/eventhub/eventhub.go +++ b/gateway/gateway-controller/pkg/eventhub/eventhub.go @@ -3,6 +3,7 @@ package eventhub import ( "context" "database/sql" + "sync" "time" "go.uber.org/zap" @@ -15,6 +16,7 @@ type eventHub struct { logger *zap.Logger initialized bool + mu sync.RWMutex } // New creates a new EventHub instance with SQLite backend (default) @@ -43,6 +45,9 @@ func NewWithBackend(backend EventhubImpl, logger *zap.Logger) EventHub { // Initialize sets up the EventHub and starts background workers func (eh *eventHub) Initialize(ctx context.Context) error { + eh.mu.Lock() + defer eh.mu.Unlock() + if eh.initialized { return nil } @@ -82,6 +87,9 @@ func (eh *eventHub) CleanUpEvents(ctx context.Context, timeFrom, timeEnd time.Ti // Close gracefully shuts down the EventHub func (eh *eventHub) Close() error { + eh.mu.Lock() + defer eh.mu.Unlock() + if !eh.initialized { return nil } diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 9b61dc8d9..4c5c49514 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -26,7 +26,7 @@ type SQLiteBackend struct { wg sync.WaitGroup initialized bool - mu sync.RWMutex + } // NewSQLiteBackend creates a new SQLite-based backend @@ -44,8 +44,6 @@ func NewSQLiteBackend(db *sql.DB, logger *zap.Logger, config *SQLiteBackendConfi // Initialize sets up the SQLite backend and starts background workers func (b *SQLiteBackend) Initialize(ctx context.Context) error { - b.mu.Lock() - defer b.mu.Unlock() if b.initialized { return nil @@ -216,8 +214,6 @@ func (b *SQLiteBackend) CleanupRange(ctx context.Context, from, to time.Time) er // Close gracefully shuts down the SQLite backend func (b *SQLiteBackend) Close() error { - b.mu.Lock() - defer b.mu.Unlock() if !b.initialized { return nil @@ -305,8 +301,16 @@ func (b *SQLiteBackend) pollAllOrganizations() { } if len(events) > 0 { - b.deliverEvents(org, events) + if len(events) > 0 { + if b.deliverEvents(org, events) != nil{ + org.updatePollState(state.VersionID, time.Now()) + } + // If delivery failed (channel full), don't update timestamp + // so events will be retried on next poll + } else { + org.updatePollState(state.VersionID, time.Now()) } + } org.updatePollState(state.VersionID, time.Now()) } @@ -339,6 +343,7 @@ func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, // getEventsSince retrieves events for an organization after a given timestamp func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since time.Time) ([]Event, error) { + // TODO: (VirajSalaka) Implement pagination if large number of events query := ` SELECT processed_timestamp, originated_timestamp, event_type, action, entity_id, correlation_id, event_data @@ -370,7 +375,7 @@ func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since } // deliverEvents sends events to all subscribers of an organization -func (b *SQLiteBackend) deliverEvents(org *organization, events []Event) { +func (b *SQLiteBackend) deliverEvents(org *organization, events []Event) error { subscribers := org.getSubscribers() if len(subscribers) == 0 { @@ -378,9 +383,10 @@ func (b *SQLiteBackend) deliverEvents(org *organization, events []Event) { zap.String("organization", string(org.id)), zap.Int("events", len(events)), ) - return + return nil } + // TODO: (VirajSalaka) One subscriber is considered here. Handle multiple subscribers properly. for _, ch := range subscribers { select { case ch <- events: @@ -389,12 +395,14 @@ func (b *SQLiteBackend) deliverEvents(org *organization, events []Event) { zap.Int("events", len(events)), ) default: - b.logger.Warn("Subscriber channel full, dropping events", + b.logger.Error("Subscriber channel full, dropping events", zap.String("organization", string(org.id)), zap.Int("events", len(events)), ) + return fmt.Errorf("subscriber channel full") } } + return nil } // cleanupLoop runs periodic cleanup of old events diff --git a/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md b/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md deleted file mode 100644 index c93e01642..000000000 --- a/gateway/gateway-controller/pkg/eventlistener/EXAMPLE_USAGE.md +++ /dev/null @@ -1,381 +0,0 @@ -# EventListener Usage Examples - -This document demonstrates how to use the EventListener with different EventSource implementations. - -## Architecture Overview - -The EventListener is designed with a generic event source abstraction: - -``` -┌─────────────────┐ -│ EventListener │ -└────────┬────────┘ - │ depends on - ▼ -┌─────────────────┐ -│ EventSource │ (interface) -└────────┬────────┘ - │ implemented by - ├─────────────────────┐ - ▼ ▼ -┌──────────────────┐ ┌────────────────┐ -│ EventHubAdapter │ │ MockEventSource│ -└──────────────────┘ └────────────────┘ - │ │ - ▼ ▼ -┌──────────────────┐ ┌────────────────┐ -│ EventHub │ │ Test Suite │ -│ (Database-based) │ │ (In-memory) │ -└──────────────────┘ └────────────────┘ -``` - -## Example 1: Using with EventHub (Production) - -```go -package main - -import ( - "context" - "database/sql" - "time" - - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" - "go.uber.org/zap" -) - -func main() { - logger, _ := zap.NewProduction() - defer logger.Sync() - - // Initialize dependencies - db := initializeDatabase() // *sql.DB - store := storage.NewConfigStore() - sqliteStorage := storage.NewSQLiteStorage(db, logger) - snapshotManager := xds.NewSnapshotManager(store, logger) - policyManager := policyxds.NewPolicyManager(logger) - routerConfig := loadRouterConfig() - - // Create EventHub - hubConfig := eventhub.DefaultConfig() - eventHub := eventhub.New(db, logger, hubConfig) - - // Initialize EventHub - ctx := context.Background() - if err := eventHub.Initialize(ctx); err != nil { - logger.Fatal("Failed to initialize EventHub", zap.Error(err)) - } - - // Wrap EventHub with adapter - eventSource := eventlistener.NewEventHubAdapter(eventHub, logger) - - // Create EventListener with the adapter - listener := eventlistener.NewEventListener( - eventSource, - store, - sqliteStorage, - snapshotManager, - policyManager, - routerConfig, - logger, - ) - - // Start listening for events - if err := listener.Start(ctx); err != nil { - logger.Fatal("Failed to start EventListener", zap.Error(err)) - } - - logger.Info("EventListener is now running") - - // ... application runs ... - - // Graceful shutdown - listener.Stop() - eventSource.Close() -} - -func initializeDatabase() *sql.DB { - // Implementation details... - return nil -} - -func loadRouterConfig() *config.RouterConfig { - // Implementation details... - return nil -} -``` - -## Example 2: Using MockEventSource for Testing - -```go -package mypackage - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" - "go.uber.org/zap" -) - -func TestEventListenerProcessesAPIEvents(t *testing.T) { - logger := zap.NewNop() - - // Create mock event source - mockSource := eventlistener.NewMockEventSource() - - // Setup test dependencies - store := setupTestStore() - db := setupTestDatabase() - snapshotManager := setupTestSnapshotManager() - routerConfig := setupTestRouterConfig() - - // Create EventListener with mock - listener := eventlistener.NewEventListener( - mockSource, - store, - db, - snapshotManager, - nil, // no policy manager for this test - routerConfig, - logger, - ) - - // Start listener - ctx := context.Background() - err := listener.Start(ctx) - assert.NoError(t, err) - - // Verify subscription was created - assert.True(t, mockSource.IsSubscribed("default")) - assert.Equal(t, 1, len(mockSource.SubscribeCalls)) - - // Publish test event - testEvent := eventlistener.Event{ - OrganizationID: "default", - EventType: "API", - Action: "CREATE", - EntityID: "test-api-123", - EventData: []byte(`{"name": "test"}`), - Timestamp: time.Now(), - } - - err = mockSource.PublishEvent("default", testEvent) - assert.NoError(t, err) - - // Allow time for processing - time.Sleep(100 * time.Millisecond) - - // Verify the event was processed - // (Check side effects in store, snapshotManager, etc.) - - // Cleanup - listener.Stop() - assert.True(t, mockSource.IsClosed()) -} - -func TestEventListenerHandlesSubscriptionError(t *testing.T) { - logger := zap.NewNop() - mockSource := eventlistener.NewMockEventSource() - - // Configure mock to return error - mockSource.SubscribeError = assert.AnError - - listener := eventlistener.NewEventListener( - mockSource, - setupTestStore(), - setupTestDatabase(), - setupTestSnapshotManager(), - nil, - setupTestRouterConfig(), - logger, - ) - - // Start should fail - ctx := context.Background() - err := listener.Start(ctx) - assert.Error(t, err) -} - -// Test helper functions -func setupTestStore() *storage.ConfigStore { /* ... */ return nil } -func setupTestDatabase() storage.Storage { /* ... */ return nil } -func setupTestSnapshotManager() *xds.SnapshotManager { /* ... */ return nil } -func setupTestRouterConfig() *config.RouterConfig { /* ... */ return nil } -``` - -## Example 3: Implementing a Custom EventSource (Kafka) - -You can implement a Kafka-based event source: - -```go -package kafka - -import ( - "context" - "encoding/json" - "sync" - - "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventlistener" - "github.com/segmentio/kafka-go" - "go.uber.org/zap" -) - -// KafkaEventSource implements EventSource using Apache Kafka -type KafkaEventSource struct { - brokers []string - logger *zap.Logger - - mu sync.RWMutex - readers map[string]*kafka.Reader - subscriptions map[string]chan<- []eventlistener.Event -} - -func NewKafkaEventSource(brokers []string, logger *zap.Logger) eventlistener.EventSource { - return &KafkaEventSource{ - brokers: brokers, - logger: logger, - readers: make(map[string]*kafka.Reader), - subscriptions: make(map[string]chan<- []eventlistener.Event), - } -} - -func (k *KafkaEventSource) Subscribe(ctx context.Context, organizationID string, eventChan chan<- []eventlistener.Event) error { - k.mu.Lock() - defer k.mu.Unlock() - - // Create Kafka reader for the organization's topic - topic := "gateway-events-" + organizationID - reader := kafka.NewReader(kafka.ReaderConfig{ - Brokers: k.brokers, - Topic: topic, - GroupID: "gateway-controller", - }) - - k.readers[organizationID] = reader - k.subscriptions[organizationID] = eventChan - - // Start consuming goroutine - go k.consumeEvents(ctx, organizationID, reader, eventChan) - - k.logger.Info("Subscribed to Kafka topic", - zap.String("organization", organizationID), - zap.String("topic", topic), - ) - - return nil -} - -func (k *KafkaEventSource) consumeEvents( - ctx context.Context, - organizationID string, - reader *kafka.Reader, - eventChan chan<- []eventlistener.Event, -) { - for { - msg, err := reader.FetchMessage(ctx) - if err != nil { - if ctx.Err() != nil { - return // Context cancelled - } - k.logger.Error("Failed to fetch Kafka message", zap.Error(err)) - continue - } - - // Parse event from Kafka message - var event eventlistener.Event - if err := json.Unmarshal(msg.Value, &event); err != nil { - k.logger.Error("Failed to unmarshal event", zap.Error(err)) - continue - } - - // Send as batch (single event) - select { - case eventChan <- []eventlistener.Event{event}: - // Successfully sent - if err := reader.CommitMessages(ctx, msg); err != nil { - k.logger.Error("Failed to commit Kafka message", zap.Error(err)) - } - case <-ctx.Done(): - return - } - } -} - -func (k *KafkaEventSource) Unsubscribe(organizationID string) error { - k.mu.Lock() - defer k.mu.Unlock() - - if reader, exists := k.readers[organizationID]; exists { - reader.Close() - delete(k.readers, organizationID) - delete(k.subscriptions, organizationID) - } - - return nil -} - -func (k *KafkaEventSource) Close() error { - k.mu.Lock() - defer k.mu.Unlock() - - for orgID, reader := range k.readers { - reader.Close() - delete(k.readers, orgID) - } - - k.subscriptions = make(map[string]chan<- []eventlistener.Event) - return nil -} -``` - -Then use it like this: - -```go -// Create Kafka event source -kafkaSource := kafka.NewKafkaEventSource( - []string{"localhost:9092"}, - logger, -) - -// Use with EventListener -listener := eventlistener.NewEventListener( - kafkaSource, - store, - db, - snapshotManager, - policyManager, - routerConfig, - logger, -) -``` - -## Benefits of this Architecture - -1. **Testability**: Easy to test EventListener with MockEventSource -2. **Flexibility**: Switch between EventHub, Kafka, RabbitMQ, etc. -3. **Decoupling**: EventListener doesn't depend on specific implementation details -4. **Maintainability**: Clear interfaces and responsibilities -5. **Extensibility**: Easy to add new event source implementations - -## Event Flow - -``` -EventSource → Subscribe(orgID, chan) → EventListener - ↓ ↓ -Publishes events Receives []Event - ↓ ↓ -Forward to channel processEvents() - ↓ ↓ -Event batches handleEvent() - ↓ - processAPIEvents() - ↓ - Update XDS & Policies -``` diff --git a/gateway/gateway-controller/pkg/eventlistener/api_processor.go b/gateway/gateway-controller/pkg/eventlistener/api_processor.go index c2a44bf53..60a16b9ba 100644 --- a/gateway/gateway-controller/pkg/eventlistener/api_processor.go +++ b/gateway/gateway-controller/pkg/eventlistener/api_processor.go @@ -84,6 +84,8 @@ func (el *EventListener) handleAPICreateOrUpdate(apiID string, correlationID str storedPolicy = el.buildStoredPolicyFromAPI(config) } + // TODO: (VirajSalaka) Handle failures in policy addition properly (rollback) + // TODO: (VirajSalaka) Use ErrGroup to parallelize XDS update and policy addition // 3. Trigger async XDS snapshot update go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go index c24ae0abd..4b5beacab 100644 --- a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go +++ b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go @@ -21,6 +21,7 @@ package eventlistener import ( "context" "fmt" + "sync" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "go.uber.org/zap" @@ -35,7 +36,7 @@ type EventHubAdapter struct { // activeSubscriptions tracks which organizations have active subscriptions // This is used to ensure proper cleanup and prevent duplicate subscriptions - activeSubscriptions map[string]chan<- []Event + activeSubscriptions sync.Map // map[string]chan<- []Event } // NewEventHubAdapter creates a new adapter that wraps an EventHub instance. @@ -48,9 +49,9 @@ type EventHubAdapter struct { // - EventSource implementation backed by EventHub func NewEventHubAdapter(eventHub eventhub.EventHub, logger *zap.Logger) EventSource { return &EventHubAdapter{ - eventHub: eventHub, - logger: logger, - activeSubscriptions: make(map[string]chan<- []Event), + eventHub: eventHub, + logger: logger, + // activeSubscriptions sync.Map zero value is ready to use } } @@ -58,7 +59,7 @@ func NewEventHubAdapter(eventHub eventhub.EventHub, logger *zap.Logger) EventSou // It handles organization registration and event conversion. func (a *EventHubAdapter) Subscribe(ctx context.Context, organizationID string, eventChan chan<- []Event) error { // Check if already subscribed - if _, exists := a.activeSubscriptions[organizationID]; exists { + if _, exists := a.activeSubscriptions.Load(organizationID); exists { return fmt.Errorf("already subscribed to organization: %s", organizationID) } @@ -81,7 +82,7 @@ func (a *EventHubAdapter) Subscribe(ctx context.Context, organizationID string, } // Track this subscription - a.activeSubscriptions[organizationID] = eventChan + a.activeSubscriptions.Store(organizationID, eventChan) // Start goroutine to convert and forward events go a.bridgeEvents(ctx, organizationID, bridgeChan, eventChan) @@ -104,7 +105,7 @@ func (a *EventHubAdapter) bridgeEvents( ) { defer func() { // Clean up subscription tracking - delete(a.activeSubscriptions, organizationID) + a.activeSubscriptions.Delete(organizationID) a.logger.Debug("Bridge goroutine exiting", zap.String("organization", organizationID), ) @@ -153,14 +154,14 @@ func (a *EventHubAdapter) bridgeEvents( // Note: The current EventHub implementation doesn't have an explicit unsubscribe method, // so we just stop the bridge goroutine by removing the subscription tracking. func (a *EventHubAdapter) Unsubscribe(organizationID string) error { - if _, exists := a.activeSubscriptions[organizationID]; !exists { + if _, exists := a.activeSubscriptions.Load(organizationID); !exists { // Not subscribed - this is fine, make it idempotent return nil } // Remove from tracking - the bridge goroutine will detect context cancellation // when the listener stops - delete(a.activeSubscriptions, organizationID) + a.activeSubscriptions.Delete(organizationID) a.logger.Info("Unsubscribed from event source", zap.String("organization", organizationID), @@ -172,9 +173,7 @@ func (a *EventHubAdapter) Unsubscribe(organizationID string) error { // Close implements EventSource.Close by delegating to EventHub.Close. func (a *EventHubAdapter) Close() error { // Clean up all subscriptions - for orgID := range a.activeSubscriptions { - _ = a.Unsubscribe(orgID) - } + a.activeSubscriptions.Clear() // Close the underlying EventHub if err := a.eventHub.Close(); err != nil { diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index 68663e7d3..3839f344e 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -174,8 +174,5 @@ CREATE TABLE IF NOT EXISTS events ( PRIMARY KEY (organization_id, processed_timestamp) ); -CREATE INDEX IF NOT EXISTS idx_events_lookup ON events(organization_id, processed_timestamp); -CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type); - -- Set schema version to 6 PRAGMA user_version = 6; diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index f28b31276..621d189df 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -246,14 +246,6 @@ func (s *SQLiteStorage) initSchema() error { return fmt.Errorf("failed to create events table: %w", err) } - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_events_lookup ON events(organization_id, processed_timestamp);`); err != nil { - return fmt.Errorf("failed to create events lookup index: %w", err) - } - - if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type);`); err != nil { - return fmt.Errorf("failed to create events type index: %w", err) - } - if _, err := s.db.Exec("PRAGMA user_version = 6"); err != nil { return fmt.Errorf("failed to set schema version to 6: %w", err) } diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index a087a87c8..2679d7381 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -30,13 +30,13 @@ import ( "github.com/google/uuid" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/generated" - policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/eventhub" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/policyxds" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/xds" + policyenginev1 "github.com/wso2/api-platform/sdk/gateway/policyengine/v1" "go.uber.org/zap" ) @@ -278,7 +278,7 @@ func (s *APIDeploymentService) DeployAPIConfiguration(params APIDeploymentParams } // Update xDS snapshot asynchronously - if (s.enableReplicaSync) { + if s.enableReplicaSync { // Multi-replica mode: Publish event to eventhub if s.eventHub != nil { // Determine action based on whether it's an update or create @@ -884,6 +884,7 @@ func convertAPIPolicy(p api.Policy) policyenginev1.PolicyInstance { } } +// TODO: (VirajSalaka) Fix working in multi-replica mode // updatePolicyConfiguration builds and updates/removes derived policy config for an API func (s *APIDeploymentService) updatePolicyConfiguration(storedCfg *models.StoredConfig, logger *zap.Logger) { if s.policyManager == nil { From a4c6a1c1d706f05d99b01bfd441370a4c3e14bc6 Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Thu, 8 Jan 2026 00:15:28 +0530 Subject: [PATCH 22/24] Resolve commented issues --- gateway/gateway-controller/cmd/controller/main.go | 5 +++-- gateway/gateway-controller/pkg/eventhub/sqlite_backend.go | 6 +----- .../pkg/storage/gateway-controller-db.sql | 6 ++++-- gateway/gateway-controller/pkg/storage/sqlite.go | 8 ++++++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index ce6f81b9b..a444f20b0 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -121,9 +121,10 @@ func main() { log.Fatal("Failed to initialize EventHub", zap.Error(err)) } if err := eventHub.RegisterOrganization("default"); err != nil { - log.Fatal("Failed to register default organization", zap.Error(err)) + log.Error("Failed to register default organization", zap.Error(err)) + } else { + log.Info("EventHub initialized successfully") } - log.Info("EventHub initialized successfully") } else { log.Fatal("EventHub requires persistent storage. Multi-replica mode will not function correctly in memory-only mode.") } diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 4c5c49514..6ca793e7d 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -301,8 +301,7 @@ func (b *SQLiteBackend) pollAllOrganizations() { } if len(events) > 0 { - if len(events) > 0 { - if b.deliverEvents(org, events) != nil{ + if b.deliverEvents(org, events) == nil{ org.updatePollState(state.VersionID, time.Now()) } // If delivery failed (channel full), don't update timestamp @@ -311,9 +310,6 @@ func (b *SQLiteBackend) pollAllOrganizations() { org.updatePollState(state.VersionID, time.Now()) } } - - org.updatePollState(state.VersionID, time.Now()) - } } // getAllStates retrieves all organization states diff --git a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql index 3839f344e..ac842aa0e 100644 --- a/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql +++ b/gateway/gateway-controller/pkg/storage/gateway-controller-db.sql @@ -169,10 +169,12 @@ CREATE TABLE IF NOT EXISTS events ( event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, - correlation_id TEXT NOT NULL DEFAULT '', + correlation_id TEXT NOT NULL, event_data TEXT NOT NULL, - PRIMARY KEY (organization_id, processed_timestamp) + PRIMARY KEY (correlation_id) ); +CREATE INDEX IF NOT EXISTS idx_events_org_time ON events(organization_id, processed_timestamp); + -- Set schema version to 6 PRAGMA user_version = 6; diff --git a/gateway/gateway-controller/pkg/storage/sqlite.go b/gateway/gateway-controller/pkg/storage/sqlite.go index 621d189df..94998e873 100644 --- a/gateway/gateway-controller/pkg/storage/sqlite.go +++ b/gateway/gateway-controller/pkg/storage/sqlite.go @@ -239,13 +239,17 @@ func (s *SQLiteStorage) initSchema() error { event_type TEXT NOT NULL, action TEXT NOT NULL CHECK(action IN ('CREATE', 'UPDATE', 'DELETE')), entity_id TEXT NOT NULL, - correlation_id TEXT NOT NULL DEFAULT '', + correlation_id TEXT NOT NULL, event_data TEXT NOT NULL, - PRIMARY KEY (organization_id, processed_timestamp) + PRIMARY KEY (correlation_id) );`); err != nil { return fmt.Errorf("failed to create events table: %w", err) } + if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_events_org_time ON events(organization_id, processed_timestamp);`); err != nil { + return fmt.Errorf("failed to create events organization-time index: %w", err) + } + if _, err := s.db.Exec("PRAGMA user_version = 6"); err != nil { return fmt.Errorf("failed to set schema version to 6: %w", err) } From 1f751568ddd934d554627eea23ba09e49e8c9b5e Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Fri, 9 Jan 2026 19:14:48 +0530 Subject: [PATCH 23/24] Resolve review comments --- gateway/gateway-controller/pkg/eventhub/sqlite_backend.go | 8 ++++++-- .../pkg/eventlistener/eventhub_adapter.go | 3 ++- gateway/gateway-controller/pkg/utils/api_deployment.go | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 6ca793e7d..521a25192 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -26,7 +26,7 @@ type SQLiteBackend struct { wg sync.WaitGroup initialized bool - + mu sync.Mutex } // NewSQLiteBackend creates a new SQLite-based backend @@ -44,6 +44,8 @@ func NewSQLiteBackend(db *sql.DB, logger *zap.Logger, config *SQLiteBackendConfi // Initialize sets up the SQLite backend and starts background workers func (b *SQLiteBackend) Initialize(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() if b.initialized { return nil @@ -215,6 +217,8 @@ func (b *SQLiteBackend) CleanupRange(ctx context.Context, from, to time.Time) er // Close gracefully shuts down the SQLite backend func (b *SQLiteBackend) Close() error { + b.mu.Lock() + defer b.mu.Unlock() if !b.initialized { return nil } @@ -301,7 +305,7 @@ func (b *SQLiteBackend) pollAllOrganizations() { } if len(events) > 0 { - if b.deliverEvents(org, events) == nil{ + if b.deliverEvents(org, events) == nil { org.updatePollState(state.VersionID, time.Now()) } // If delivery failed (channel full), don't update timestamp diff --git a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go index 4b5beacab..7e5f00593 100644 --- a/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go +++ b/gateway/gateway-controller/pkg/eventlistener/eventhub_adapter.go @@ -128,13 +128,14 @@ func (a *EventHubAdapter) bridgeEvents( // Convert eventhub.Event to generic Event genericEvents := make([]Event, len(hubEvents)) for i, hubEvent := range hubEvents { + eventCorrelationID := "event-" + hubEvent.CorrelationID genericEvents[i] = Event{ OrganizationID: string(hubEvent.OrganizationID), EventType: string(hubEvent.EventType), Action: hubEvent.Action, EntityID: hubEvent.EntityID, EventData: hubEvent.EventData, - CorrelationID: hubEvent.CorrelationID, + CorrelationID: eventCorrelationID, Timestamp: hubEvent.ProcessedTimestamp, } } diff --git a/gateway/gateway-controller/pkg/utils/api_deployment.go b/gateway/gateway-controller/pkg/utils/api_deployment.go index 2679d7381..09ab2fb02 100644 --- a/gateway/gateway-controller/pkg/utils/api_deployment.go +++ b/gateway/gateway-controller/pkg/utils/api_deployment.go @@ -567,6 +567,8 @@ func (s *APIDeploymentService) saveOrUpdateConfig(storedCfg *models.StoredConfig } // Try to add to in-memory store + // Multi-replica mode: in-memory store will be updated via EventListener + // after event processing to ensure consistency across all replicas if !s.enableReplicaSync { if err := s.store.Add(storedCfg); err != nil { // Check if it's a conflict (API already exists) From 48f0f0f71db41e65a4e58b11267071d0b7a1883e Mon Sep 17 00:00:00 2001 From: VirajSalaka Date: Wed, 14 Jan 2026 11:10:32 +0530 Subject: [PATCH 24/24] Improve DB connections to optimize querying --- .../pkg/eventhub/sqlite_backend.go | 412 ++++++++++++++++-- 1 file changed, 369 insertions(+), 43 deletions(-) diff --git a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go index 521a25192..83b230c66 100644 --- a/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go +++ b/gateway/gateway-controller/pkg/eventhub/sqlite_backend.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sync" "time" @@ -11,6 +12,19 @@ import ( "go.uber.org/zap" ) +// statementKey identifies a prepared statement for re-preparation +type statementKey int + +const ( + stmtKeyGetAllStates statementKey = iota + stmtKeyGetEventsSince + stmtKeyInsertEvent + stmtKeyUpdateOrgState + stmtKeyInsertOrgState + stmtKeyCleanup + stmtKeyCleanupRange +) + // SQLiteBackend implements the Backend interface using SQLite with polling type SQLiteBackend struct { db *sql.DB @@ -25,8 +39,17 @@ type SQLiteBackend struct { cleanupCancel context.CancelFunc wg sync.WaitGroup + // Prepared statements for performance + stmtGetAllStates *sql.Stmt + stmtGetEventsSince *sql.Stmt + stmtInsertEvent *sql.Stmt + stmtUpdateOrgState *sql.Stmt + stmtInsertOrgState *sql.Stmt + stmtCleanup *sql.Stmt + stmtCleanupRange *sql.Stmt + initialized bool - mu sync.Mutex + mu sync.RWMutex } // NewSQLiteBackend creates a new SQLite-based backend @@ -53,6 +76,11 @@ func (b *SQLiteBackend) Initialize(ctx context.Context) error { b.logger.Info("Initializing SQLite backend") + // Prepare statements for performance + if err := b.prepareStatements(); err != nil { + return fmt.Errorf("failed to prepare statements: %w", err) + } + // Start poller b.pollerCtx, b.pollerCancel = context.WithCancel(ctx) b.wg.Add(1) @@ -72,21 +100,293 @@ func (b *SQLiteBackend) Initialize(ctx context.Context) error { return nil } +// ensureInitialized checks if the backend is initialized and returns an error if not +func (b *SQLiteBackend) ensureInitialized() error { + b.mu.RLock() + defer b.mu.RUnlock() + + if !b.initialized { + return fmt.Errorf("SQLite backend not initialized") + } + return nil +} + +// getStatement returns the prepared statement for the given key (caller must hold at least RLock) +func (b *SQLiteBackend) getStatement(key statementKey) *sql.Stmt { + switch key { + case stmtKeyGetAllStates: + return b.stmtGetAllStates + case stmtKeyGetEventsSince: + return b.stmtGetEventsSince + case stmtKeyInsertEvent: + return b.stmtInsertEvent + case stmtKeyUpdateOrgState: + return b.stmtUpdateOrgState + case stmtKeyInsertOrgState: + return b.stmtInsertOrgState + case stmtKeyCleanup: + return b.stmtCleanup + case stmtKeyCleanupRange: + return b.stmtCleanupRange + default: + return nil + } +} + +// setStatement sets the prepared statement for the given key (caller must hold Lock) +func (b *SQLiteBackend) setStatement(key statementKey, stmt *sql.Stmt) { + switch key { + case stmtKeyGetAllStates: + b.stmtGetAllStates = stmt + case stmtKeyGetEventsSince: + b.stmtGetEventsSince = stmt + case stmtKeyInsertEvent: + b.stmtInsertEvent = stmt + case stmtKeyUpdateOrgState: + b.stmtUpdateOrgState = stmt + case stmtKeyInsertOrgState: + b.stmtInsertOrgState = stmt + case stmtKeyCleanup: + b.stmtCleanup = stmt + case stmtKeyCleanupRange: + b.stmtCleanupRange = stmt + } +} + +// prepareStatement prepares a single statement by key +func (b *SQLiteBackend) prepareStatement(key statementKey) (*sql.Stmt, error) { + var query string + switch key { + case stmtKeyGetAllStates: + query = ` + SELECT organization, version_id, updated_at + FROM organization_states + ORDER BY organization + ` + case stmtKeyGetEventsSince: + query = ` + SELECT processed_timestamp, originated_timestamp, event_type, + action, entity_id, correlation_id, event_data + FROM events + WHERE organization_id = ? AND processed_timestamp > ? + ORDER BY processed_timestamp ASC + ` + case stmtKeyInsertEvent: + query = ` + INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, + event_type, action, entity_id, correlation_id, event_data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ` + case stmtKeyUpdateOrgState: + query = ` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at + ` + case stmtKeyInsertOrgState: + query = ` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO NOTHING + ` + case stmtKeyCleanup: + query = `DELETE FROM events WHERE processed_timestamp < ?` + case stmtKeyCleanupRange: + query = `DELETE FROM events WHERE processed_timestamp >= ? AND processed_timestamp <= ?` + default: + return nil, fmt.Errorf("unknown statement key: %d", key) + } + + stmt, err := b.db.Prepare(query) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement (key=%d): %w", key, err) + } + return stmt, nil +} + +// isRecoverableError checks if an error indicates a statement needs re-preparation +func (b *SQLiteBackend) isRecoverableError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + // SQLite schema change errors indicate statements need re-preparation + return strings.Contains(errStr, "schema") || + strings.Contains(errStr, "SQLITE_SCHEMA") +} + +// execWithRetry executes a prepared statement with automatic re-preparation on recoverable errors +func (b *SQLiteBackend) execWithRetry(ctx context.Context, key statementKey, args ...any) (sql.Result, error) { + b.mu.RLock() + stmt := b.getStatement(key) + b.mu.RUnlock() + + if stmt == nil { + return nil, fmt.Errorf("statement not initialized (key=%d)", key) + } + + result, err := stmt.ExecContext(ctx, args...) + if err != nil && b.isRecoverableError(err) { + // Re-prepare and retry once + b.logger.Warn("Statement execution failed with recoverable error, re-preparing", + zap.Int("statementKey", int(key)), + zap.Error(err)) + + b.mu.Lock() + newStmt, prepErr := b.prepareStatement(key) + if prepErr == nil { + // Close old statement + if oldStmt := b.getStatement(key); oldStmt != nil { + _ = oldStmt.Close() + } + b.setStatement(key, newStmt) + stmt = newStmt + } + b.mu.Unlock() + + if prepErr != nil { + return nil, fmt.Errorf("re-preparation failed after recoverable error: %w (original: %v)", prepErr, err) + } + + // Retry with new statement + result, err = stmt.ExecContext(ctx, args...) + } + return result, err +} + +// queryWithRetry executes a prepared query with automatic re-preparation on recoverable errors +func (b *SQLiteBackend) queryWithRetry(ctx context.Context, key statementKey, args ...any) (*sql.Rows, error) { + b.mu.RLock() + stmt := b.getStatement(key) + b.mu.RUnlock() + + if stmt == nil { + return nil, fmt.Errorf("statement not initialized (key=%d)", key) + } + + rows, err := stmt.QueryContext(ctx, args...) + if err != nil && b.isRecoverableError(err) { + // Re-prepare and retry once + b.logger.Warn("Statement query failed with recoverable error, re-preparing", + zap.Int("statementKey", int(key)), + zap.Error(err)) + + b.mu.Lock() + newStmt, prepErr := b.prepareStatement(key) + if prepErr == nil { + // Close old statement + if oldStmt := b.getStatement(key); oldStmt != nil { + _ = oldStmt.Close() + } + b.setStatement(key, newStmt) + stmt = newStmt + } + b.mu.Unlock() + + if prepErr != nil { + return nil, fmt.Errorf("re-preparation failed after recoverable error: %w (original: %v)", prepErr, err) + } + + // Retry with new statement + rows, err = stmt.QueryContext(ctx, args...) + } + return rows, err +} + +// prepareStatements prepares all frequently-used SQL statements for performance +func (b *SQLiteBackend) prepareStatements() (err error) { + // Clean up any successfully prepared statements if we fail partway through + defer func() { + if err != nil { + b.closeStatements() + } + }() + + // Prepare getAllStates query + b.stmtGetAllStates, err = b.db.Prepare(` + SELECT organization, version_id, updated_at + FROM organization_states + ORDER by organization + `) + if err != nil { + return fmt.Errorf("failed to prepare getAllStates: %w", err) + } + + // Prepare getEventsSince query + b.stmtGetEventsSince, err = b.db.Prepare(` + SELECT processed_timestamp, originated_timestamp, event_type, + action, entity_id, correlation_id, event_data + FROM events + WHERE organization_id = ? AND processed_timestamp > ? + ORDER BY processed_timestamp ASC + `) + if err != nil { + return fmt.Errorf("failed to prepare getEventsSince: %w", err) + } + + // Prepare insertEvent query + b.stmtInsertEvent, err = b.db.Prepare(` + INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, + event_type, action, entity_id, correlation_id, event_data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return fmt.Errorf("failed to prepare insertEvent: %w", err) + } + + // Prepare updateOrgState query + b.stmtUpdateOrgState, err = b.db.Prepare(` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at + `) + if err != nil { + return fmt.Errorf("failed to prepare updateOrgState: %w", err) + } + + // Prepare insertOrgState query (for RegisterOrganization) + b.stmtInsertOrgState, err = b.db.Prepare(` + INSERT INTO organization_states (organization, version_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(organization) + DO NOTHING + `) + if err != nil { + return fmt.Errorf("failed to prepare insertOrgState: %w", err) + } + + // Prepare cleanup query + b.stmtCleanup, err = b.db.Prepare(`DELETE FROM events WHERE processed_timestamp < ?`) + if err != nil { + return fmt.Errorf("failed to prepare cleanup: %w", err) + } + + // Prepare cleanupRange query + b.stmtCleanupRange, err = b.db.Prepare(`DELETE FROM events WHERE processed_timestamp >= ? AND processed_timestamp <= ?`) + if err != nil { + return fmt.Errorf("failed to prepare cleanupRange: %w", err) + } + + b.logger.Info("Prepared statements initialized successfully") + return nil +} + // RegisterOrganization creates the necessary resources for an organization func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID string) error { + if err := b.ensureInitialized(); err != nil { + return err + } + // Register in local registry if err := b.registry.register(orgID); err != nil { return err } - // Initialize state in database - query := ` - INSERT INTO organization_states (organization, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(organization) - DO NOTHING - ` - _, err := b.db.ExecContext(ctx, query, string(orgID), "", time.Now()) + // Initialize state in database using prepared statement with retry + _, err := b.execWithRetry(ctx, stmtKeyInsertOrgState, string(orgID), "", time.Now()) if err != nil { return fmt.Errorf("failed to initialize organization state: %w", err) } @@ -101,6 +401,10 @@ func (b *SQLiteBackend) RegisterOrganization(ctx context.Context, orgID string) func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, eventType EventType, action, entityID, correlationID string, eventData []byte) error { + if err := b.ensureInitialized(); err != nil { + return err + } + // Verify organization is registered _, err := b.registry.get(orgID) if err != nil { @@ -116,27 +420,24 @@ func (b *SQLiteBackend) Publish(ctx context.Context, orgID string, now := time.Now() - // Insert event - insertQuery := ` - INSERT INTO events (organization_id, processed_timestamp, originated_timestamp, - event_type, action, entity_id, correlation_id, event_data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ` - _, err = tx.ExecContext(ctx, insertQuery, + // Use prepared statements within transaction + // Note: For transaction-bound statements, we use tx.Stmt() to get transaction-specific handles + // Retry logic doesn't apply within transactions as the transaction would need to be restarted + b.mu.RLock() + txStmtInsertEvent := tx.Stmt(b.stmtInsertEvent) + txStmtUpdateOrgState := tx.Stmt(b.stmtUpdateOrgState) + b.mu.RUnlock() + + // Insert event using prepared statement + _, err = txStmtInsertEvent.ExecContext(ctx, string(orgID), now, now, string(eventType), action, entityID, correlationID, eventData) if err != nil { return fmt.Errorf("failed to record event: %w", err) } - // Update organization state version + // Update organization state version using prepared statement newVersion := uuid.New().String() - updateQuery := ` - INSERT INTO organization_states (organization, version_id, updated_at) - VALUES (?, ?, ?) - ON CONFLICT(organization) - DO UPDATE SET version_id = excluded.version_id, updated_at = excluded.updated_at - ` - _, err = tx.ExecContext(ctx, updateQuery, string(orgID), newVersion, now) + _, err = txStmtUpdateOrgState.ExecContext(ctx, string(orgID), newVersion, now) if err != nil { return fmt.Errorf("failed to update state: %w", err) } @@ -183,8 +484,11 @@ func (b *SQLiteBackend) Unsubscribe(orgID string, eventChan chan<- []Event) erro // Cleanup removes old events based on retention policy func (b *SQLiteBackend) Cleanup(ctx context.Context, olderThan time.Time) error { - query := `DELETE FROM events WHERE processed_timestamp < ?` - result, err := b.db.ExecContext(ctx, query, olderThan) + if err := b.ensureInitialized(); err != nil { + return err + } + + result, err := b.execWithRetry(ctx, stmtKeyCleanup, olderThan) if err != nil { return fmt.Errorf("failed to cleanup old events: %w", err) } @@ -199,8 +503,11 @@ func (b *SQLiteBackend) Cleanup(ctx context.Context, olderThan time.Time) error // CleanupRange removes events within a specific time range func (b *SQLiteBackend) CleanupRange(ctx context.Context, from, to time.Time) error { - query := `DELETE FROM events WHERE processed_timestamp >= ? AND processed_timestamp <= ?` - result, err := b.db.ExecContext(ctx, query, from, to) + if err := b.ensureInitialized(); err != nil { + return err + } + + result, err := b.execWithRetry(ctx, stmtKeyCleanupRange, from, to) if err != nil { return fmt.Errorf("failed to cleanup events: %w", err) } @@ -238,11 +545,36 @@ func (b *SQLiteBackend) Close() error { // Wait for goroutines b.wg.Wait() + // Close all prepared statements + b.closeStatements() + b.initialized = false b.logger.Info("SQLite backend shutdown complete") return nil } +// closeStatements closes all prepared statements +func (b *SQLiteBackend) closeStatements() { + statements := []*sql.Stmt{ + b.stmtGetAllStates, + b.stmtGetEventsSince, + b.stmtInsertEvent, + b.stmtUpdateOrgState, + b.stmtInsertOrgState, + b.stmtCleanup, + b.stmtCleanupRange, + } + + for _, stmt := range statements { + if stmt != nil { + if err := stmt.Close(); err != nil { + b.logger.Warn("Failed to close prepared statement", zap.Error(err)) + } + } + } + b.logger.Info("Prepared statements closed successfully") +} + // pollLoop runs the main polling loop for state changes func (b *SQLiteBackend) pollLoop() { defer b.wg.Done() @@ -318,13 +650,11 @@ func (b *SQLiteBackend) pollAllOrganizations() { // getAllStates retrieves all organization states func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, error) { - query := ` - SELECT organization, version_id, updated_at - FROM organization_states - ORDER BY organization - ` + if err := b.ensureInitialized(); err != nil { + return nil, err + } - rows, err := b.db.QueryContext(ctx, query) + rows, err := b.queryWithRetry(ctx, stmtKeyGetAllStates) if err != nil { return nil, fmt.Errorf("failed to query all states: %w", err) } @@ -343,16 +673,12 @@ func (b *SQLiteBackend) getAllStates(ctx context.Context) ([]OrganizationState, // getEventsSince retrieves events for an organization after a given timestamp func (b *SQLiteBackend) getEventsSince(ctx context.Context, orgID string, since time.Time) ([]Event, error) { - // TODO: (VirajSalaka) Implement pagination if large number of events - query := ` - SELECT processed_timestamp, originated_timestamp, event_type, - action, entity_id, correlation_id, event_data - FROM events - WHERE organization_id = ? AND processed_timestamp > ? - ORDER BY processed_timestamp ASC - ` + if err := b.ensureInitialized(); err != nil { + return nil, err + } - rows, err := b.db.QueryContext(ctx, query, string(orgID), since) + // TODO: (VirajSalaka) Implement pagination if large number of events + rows, err := b.queryWithRetry(ctx, stmtKeyGetEventsSince, string(orgID), since) if err != nil { return nil, fmt.Errorf("failed to query events: %w", err) }