From 4c367118cfd0b0dfb397a31679dde13d2db2c888 Mon Sep 17 00:00:00 2001 From: CDevmina Date: Wed, 21 May 2025 02:25:59 +0530 Subject: [PATCH 1/3] feat: Implement preference access logging and update related data structures --- .../service/StoreOperationsService.js | 27 +++++++++++--- tapiro-api-internal/api/openapi.yaml | 20 ++++++++--- tapiro-api-internal/utils/dbSchemas.js | 6 ++-- web/src/api/types/data-contracts.ts | 19 +++++++--- .../pages/UserDashboard/UserAnalyticsPage.tsx | 36 +++++++++++++++++-- web/src/pages/UserDashboard/UserDashboard.tsx | 15 +++++--- 6 files changed, 99 insertions(+), 24 deletions(-) diff --git a/tapiro-api-external/service/StoreOperationsService.js b/tapiro-api-external/service/StoreOperationsService.js index 0edf0a7..0395ece 100644 --- a/tapiro-api-external/service/StoreOperationsService.js +++ b/tapiro-api-external/service/StoreOperationsService.js @@ -23,11 +23,8 @@ exports.getUserPreferences = async function (req, userId) { // Try cache first using preference-specific cache key with constants const cacheKey = `${CACHE_KEYS.STORE_PREFERENCES}${userId}:${req.storeId}`; const cachedPrefs = await getCache(cacheKey); - if (cachedPrefs) { - return respondWithCode(200, JSON.parse(cachedPrefs)); - } - - // Find user in database by email only + + // Find user in database by email only (needed regardless of cache hit/miss for logging) const user = await db.collection('users').findOne({ email: userId }); if (!user) { @@ -54,6 +51,26 @@ exports.getUserPreferences = async function (req, userId) { }); } + // Log preference access event regardless of cache hit/miss + await db.collection('userData').insertOne({ + userId: user._id, + storeId: req.storeId, + email: userId, + dataType: 'preference_access', // New data type for preference access + entries: [{ timestamp: new Date() }], // Simplified entry structure for preference access + metadata: { + source: req.keyId ? `api-key:${req.keyId}` : 'api', + userAgent: req.headers['user-agent'] || 'unknown', + }, + processedStatus: 'processed', // No need for AI processing for preference access + timestamp: new Date(), + }); + + // Return cached response if available + if (cachedPrefs) { + return respondWithCode(200, JSON.parse(cachedPrefs)); + } + // Check if user has explicitly opted in to this store const isOptedIn = user.privacySettings?.optInStores?.includes(req.storeId); diff --git a/tapiro-api-internal/api/openapi.yaml b/tapiro-api-internal/api/openapi.yaml index 0791cba..f80e4c4 100644 --- a/tapiro-api-internal/api/openapi.yaml +++ b/tapiro-api-internal/api/openapi.yaml @@ -641,11 +641,11 @@ paths: default: 1 - name: dataType in: query - description: Filter by data type (purchase or search) + description: Filter by data type (purchase, search, or preference_access) required: false schema: type: string - enum: [purchase, search] + enum: [purchase, search, preference_access] - name: storeId in: query description: Filter by store ID @@ -1596,6 +1596,17 @@ components: items: type: string + PreferenceAccessEntry: + type: object + required: + - timestamp + properties: + timestamp: + type: string + format: date-time + description: ISO 8601 timestamp of when the preferences were accessed. + description: Record of when a store accessed a user's preference data. + RecentUserDataEntry: type: object properties: @@ -1607,7 +1618,7 @@ components: description: The ID of the store that submitted the data. dataType: type: string - enum: [purchase, search] + enum: [purchase, search, preference_access] description: The type of data submitted. timestamp: type: string @@ -1619,11 +1630,12 @@ components: description: The timestamp of the original event (e.g., purchase time). details: type: array - description: Array of individual purchase or search events within this batch. + description: Array of individual purchase, search, or preference access events within this batch. items: oneOf: - $ref: "#/components/schemas/PurchaseEntry" - $ref: "#/components/schemas/SearchEntry" + - $ref: "#/components/schemas/PreferenceAccessEntry" StoreBasicInfo: type: object diff --git a/tapiro-api-internal/utils/dbSchemas.js b/tapiro-api-internal/utils/dbSchemas.js index afb11a7..f9d66db 100644 --- a/tapiro-api-internal/utils/dbSchemas.js +++ b/tapiro-api-internal/utils/dbSchemas.js @@ -3,7 +3,7 @@ */ // Schema version tracking -const SCHEMA_VERSION = '3.0.0'; // Incremented version +const SCHEMA_VERSION = '3.0.1'; // Incremented version const userSchema = { validator: { @@ -255,8 +255,8 @@ const userDataSchema = { email: { bsonType: 'string' }, dataType: { bsonType: 'string', - enum: ['purchase', 'search'], - description: 'Type of data being stored', + enum: ['purchase', 'search', 'preference_access'], + description: 'Type of data collected', }, entries: { bsonType: 'array', diff --git a/web/src/api/types/data-contracts.ts b/web/src/api/types/data-contracts.ts index 4978a37..e173752 100644 --- a/web/src/api/types/data-contracts.ts +++ b/web/src/api/types/data-contracts.ts @@ -484,13 +484,22 @@ export interface SearchEntry { clicked?: string[] | null; } +/** Record of when a store accessed a user's preference data. */ +export interface PreferenceAccessEntry { + /** + * ISO 8601 timestamp of when the preferences were accessed. + * @format date-time + */ + timestamp: string; +} + export interface RecentUserDataEntry { /** The unique ID of the userData entry. */ _id?: string; /** The ID of the store that submitted the data. */ storeId?: string; /** The type of data submitted. */ - dataType?: "purchase" | "search"; + dataType?: "purchase" | "search" | "preference_access"; /** * When the data was submitted to Tapiro. * @format date-time @@ -501,8 +510,8 @@ export interface RecentUserDataEntry { * @format date-time */ entryTimestamp?: string; - /** Array of individual purchase or search events within this batch. */ - details?: (PurchaseEntry | SearchEntry)[]; + /** Array of individual purchase, search, or preference access events within this batch. */ + details?: (PurchaseEntry | SearchEntry | PreferenceAccessEntry)[]; } export interface StoreBasicInfo { @@ -605,8 +614,8 @@ export interface GetRecentUserDataParams { * @default 1 */ page?: number; - /** Filter by data type (purchase or search) */ - dataType?: "purchase" | "search"; + /** Filter by data type (purchase, search, or preference_access) */ + dataType?: "purchase" | "search" | "preference_access"; /** Filter by store ID */ storeId?: string; /** diff --git a/web/src/pages/UserDashboard/UserAnalyticsPage.tsx b/web/src/pages/UserDashboard/UserAnalyticsPage.tsx index 7b64ba5..3402498 100644 --- a/web/src/pages/UserDashboard/UserAnalyticsPage.tsx +++ b/web/src/pages/UserDashboard/UserAnalyticsPage.tsx @@ -45,6 +45,7 @@ import { HiCheckCircle, // For success toast HiXCircle, // For error toast HiOutlineEye, // Added Eye icon for view details + HiOutlineSparkles, // Added Sparkles icon for preference access } from "react-icons/hi"; import { useRecentUserData, @@ -503,7 +504,9 @@ const UserAnalyticsPage: React.FC = () => { // Cast value back to the correct type or handle empty string const value = e.target.value; setActivityDataType( - value === "purchase" || value === "search" + value === "purchase" || + value === "search" || + value === "preference_access" ? value : undefined, ); @@ -513,6 +516,7 @@ const UserAnalyticsPage: React.FC = () => { +
@@ -599,7 +603,9 @@ const UserAnalyticsPage: React.FC = () => { {formatDate(entry.timestamp)} - {entry.dataType} + {entry.dataType === "preference_access" + ? "Preference Access" + : entry.dataType} {entry.storeId @@ -622,6 +628,9 @@ const UserAnalyticsPage: React.FC = () => { searchDetail?.query && ( Query: "{searchDetail.query}" )} + {entry.dataType === "preference_access" && ( + Store accessed your preference data + )}
)} + {selectedEntryForDetails.dataType === "preference_access" && ( +
+
+
+ +

+ Preference Data Access +

+
+

+ {storeNameMap.get(selectedEntryForDetails.storeId) || + "A store"}{" "} + accessed your preference data. This allows them to + personalize your shopping experience based on your + interests. +

+

+ You can manage which stores can access your data in the + "Sharing" tab. +

+
+
+ )} ))} {!selectedEntryForDetails.details || diff --git a/web/src/pages/UserDashboard/UserDashboard.tsx b/web/src/pages/UserDashboard/UserDashboard.tsx index d1bfa4b..f8e3fa6 100644 --- a/web/src/pages/UserDashboard/UserDashboard.tsx +++ b/web/src/pages/UserDashboard/UserDashboard.tsx @@ -504,17 +504,22 @@ export default function UserDashboard() { {recentActivity.map( (entry: RecentUserDataEntry) => ( - + {formatDate(entry.timestamp)} - {entry.dataType} - {entry.storeId && - ` at ${storeNameMap.get(entry.storeId) || "Unknown Store"}`} + {entry.dataType === "preference_access" + ? `Preferences accessed by ${storeNameMap.get(entry.storeId) || "Unknown Store"}` + : `${entry.dataType}${entry.storeId ? ` at ${storeNameMap.get(entry.storeId) || "Unknown Store"}` : ""}`} - {/* Further details can be added here if needed */} ), From 1780739cf02fd80fde01c22dea2faee878f5549f Mon Sep 17 00:00:00 2001 From: CDevmina Date: Wed, 21 May 2025 02:30:24 +0530 Subject: [PATCH 2/3] feat: Update document title to reflect centralized data management --- web/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/index.html b/web/index.html index c705723..b35ba95 100644 --- a/web/index.html +++ b/web/index.html @@ -4,7 +4,7 @@ - Flowbite React Template React Router - Data mode + Tapiro: Centralized Data Management
From 3ae4fd141978441052002a28b88464e47d4ead06 Mon Sep 17 00:00:00 2001 From: CDevmina Date: Wed, 21 May 2025 03:06:09 +0530 Subject: [PATCH 3/3] feat: Enhance API key validation with revocation checks and improve cache handling --- .../middleware/apiKeyMiddleware.js | 47 ++++++++++++------- tapiro-api-external/utils/cacheConfig.js | 1 + .../service/StoreManagementService.js | 24 ++++++---- tapiro-api-internal/utils/cacheConfig.js | 1 + .../pages/UserDashboard/UserAnalyticsPage.tsx | 15 +++--- web/src/pages/UserDashboard/UserDashboard.tsx | 2 +- 6 files changed, 58 insertions(+), 32 deletions(-) diff --git a/tapiro-api-external/middleware/apiKeyMiddleware.js b/tapiro-api-external/middleware/apiKeyMiddleware.js index d12a5eb..2a30faa 100644 --- a/tapiro-api-external/middleware/apiKeyMiddleware.js +++ b/tapiro-api-external/middleware/apiKeyMiddleware.js @@ -41,19 +41,38 @@ const validateApiKey = async (req, scopes, schema) => { throw new Error('API key required'); } - const apiKeyDetailsCacheKey = `${CACHE_KEYS.API_KEY_DETAILS}${apiKey}`; // New cache key + const apiKeyDetailsCacheKey = `${CACHE_KEYS.API_KEY_DETAILS}${apiKey}`; let cachedApiKeyDetails = await getCache(apiKeyDetailsCacheKey); if (cachedApiKeyDetails) { cachedApiKeyDetails = JSON.parse(cachedApiKeyDetails); - if (cachedApiKeyDetails.status === 'active') { - req.storeId = cachedApiKeyDetails.storeId; - req.keyId = cachedApiKeyDetails.keyId; // Set keyId for tracking + const { keyId: cachedKeyId, storeId: cachedStoreId, status: cachedStatus } = cachedApiKeyDetails; + + if (cachedStatus === 'active') { + // Check for an explicit revocation marker for this keyId + let isMarkedRevoked = false; + if (cachedKeyId) { + const markerKey = `revoked_api_key_marker:${cachedKeyId}`; + const revocationMarker = await getCache(markerKey); + if (revocationMarker === 'revoked') { + isMarkedRevoked = true; + } + } + + if (isMarkedRevoked) { + console.log(`API key ${apiKey.substring(0,8)}... (keyId: ${cachedKeyId}) found revocation marker. Invalidating local cache.`); + await invalidateCache(apiKeyDetailsCacheKey); // Invalidate this specific raw API key's cache + throw new Error('API key revoked or invalid (marker)'); + } + + // If not marked revoked, proceed + req.storeId = cachedStoreId; + req.keyId = cachedKeyId; trackApiUsage(req, apiKey, req.storeId, req.keyId); return true; } else { - // Key was cached but is not active (e.g. revoked) - throw new Error('API key revoked or invalid'); + // Key was cached but is not active (e.g. revoked directly in this cache) + throw new Error('API key revoked or invalid (cached as non-active)'); } } @@ -63,7 +82,7 @@ const validateApiKey = async (req, scopes, schema) => { const store = await db.collection('stores').findOne({ 'apiKeys.prefix': prefix, - 'apiKeys.status': 'active', // Query for active keys directly + 'apiKeys.status': 'active', }); if (!store) { @@ -75,8 +94,6 @@ const validateApiKey = async (req, scopes, schema) => { ); if (!foundKey) { - // This case should ideally be covered by the store query if prefix is unique enough - // and status is checked. throw new Error('Invalid API key (specific key not found or inactive)'); } @@ -86,26 +103,22 @@ const validateApiKey = async (req, scopes, schema) => { throw new Error('Invalid API key (hash mismatch)'); } - // Set store ID and key ID in request req.storeId = store._id.toString(); req.keyId = foundKey.keyId; - // Cache the API key details (storeId, keyId, status) const apiKeyDetailsToCache = { storeId: req.storeId, keyId: req.keyId, - status: foundKey.status, // Should be 'active' here + status: foundKey.status, }; - await setCache(apiKeyDetailsCacheKey, JSON.stringify(apiKeyDetailsToCache), { EX: CACHE_TTL.API_KEY || 1800 }); + // Use the correct TTL from local cacheConfig for API_KEY_DETAILS + await setCache(apiKeyDetailsCacheKey, JSON.stringify(apiKeyDetailsToCache), { EX: CACHE_TTL.API_KEY_DETAILS || 1800 }); trackApiUsage(req, apiKey, req.storeId, req.keyId); return true; } catch (error) { - console.error('API key validation failed:', error.message); // Log only message for brevity - // Re-throw to be handled by the oas3-tools error handler or a global error handler - // which should return a proper HTTP error response. - // Avoid directly sending res.status here as it bypasses standard error flow. + console.error('API key validation failed:', error.message); throw error; } }; diff --git a/tapiro-api-external/utils/cacheConfig.js b/tapiro-api-external/utils/cacheConfig.js index c25c0d3..7c2f86b 100644 --- a/tapiro-api-external/utils/cacheConfig.js +++ b/tapiro-api-external/utils/cacheConfig.js @@ -7,6 +7,7 @@ const CACHE_TTL = { USER_DATA: 3600, // User profiles - 1 hour STORE_DATA: 3600, // Store profiles - 1 hour API_KEY: 1800, // API keys - 30 minutes + API_KEY_DETAILS: 1800, // TTL for detailed API key info - 30 minutes INVALIDATION: 1, // Short TTL for invalidation AI_REQUEST: 60, // AI service requests - 1 minute }; diff --git a/tapiro-api-internal/service/StoreManagementService.js b/tapiro-api-internal/service/StoreManagementService.js index 690b46f..01af308 100644 --- a/tapiro-api-internal/service/StoreManagementService.js +++ b/tapiro-api-internal/service/StoreManagementService.js @@ -1,10 +1,10 @@ -const crypto = require('crypto'); -const { ObjectId } = require('mongodb'); const { getDB } = require('../utils/mongoUtil'); -const { respondWithCode } = require('../utils/writer'); const { setCache, getCache, invalidateCache } = require('../utils/redisUtil'); +const { respondWithCode } = require('../utils/writer'); const { getUserData } = require('../utils/authUtil'); const { CACHE_TTL, CACHE_KEYS } = require('../utils/cacheConfig'); +const { ObjectId } = require('mongodb'); +const crypto = require('crypto'); /** * Create API Key @@ -145,9 +145,7 @@ exports.getApiKeys = async function (req) { exports.revokeApiKey = async function (req, keyId) { try { const db = getDB(); - - // Get user data - use req.user if available (from middleware) or fetch it - const userData = req.user || await getUserData(req.headers.authorization?.split(' ')[1]); + const userData = req.user || (await getUserData(req.headers.authorization?.split(' ')[1])); // Get the store with API keys first to find the prefix of the key being revoked const store = await db.collection('stores').findOne({ auth0Id: userData.sub }); @@ -191,12 +189,22 @@ exports.revokeApiKey = async function (req, keyId) { if (result.matchedCount === 0) { return respondWithCode(404, { code: 404, - message: 'Store not found', + message: 'Store not found or API key not matched during update', }); } - // Invalidate store cache to reflect modified API key + // Invalidate store cache to reflect modified API key for the store owner's view await invalidateCache(`${CACHE_KEYS.STORE_DATA}${userData.sub}`); + + // Set a revocation marker for the specific keyId. + // The external API will check for this marker. + const markerKey = `revoked_api_key_marker:${keyToRevoke.keyId}`; + const externalApiKeyCacheTTL = CACHE_TTL.EXTERNAL_API_KEY_DETAILS || 1800; // Use the new constant + const markerBuffer = 300; // 5 minutes buffer + const markerTTL = externalApiKeyCacheTTL + markerBuffer; + + await setCache(markerKey, 'revoked', { EX: markerTTL }); + console.log(`Set revocation marker for keyId ${keyToRevoke.keyId} with TTL ${markerTTL}s. Marker key: ${markerKey}`); return respondWithCode(204); } catch (error) { diff --git a/tapiro-api-internal/utils/cacheConfig.js b/tapiro-api-internal/utils/cacheConfig.js index 32dfdad..3ffd032 100644 --- a/tapiro-api-internal/utils/cacheConfig.js +++ b/tapiro-api-internal/utils/cacheConfig.js @@ -10,6 +10,7 @@ const CACHE_TTL = { INVALIDATION: 1, // Short TTL for invalidation AI_REQUEST: 60, // AI service requests - 1 minute TAXONOMY: 86400, // Taxonomy data - 1 day (added) + EXTERNAL_API_KEY_DETAILS: 1800, // TTL for API key details in the external API - 30 minutes }; /** diff --git a/web/src/pages/UserDashboard/UserAnalyticsPage.tsx b/web/src/pages/UserDashboard/UserAnalyticsPage.tsx index 3402498..c1dd169 100644 --- a/web/src/pages/UserDashboard/UserAnalyticsPage.tsx +++ b/web/src/pages/UserDashboard/UserAnalyticsPage.tsx @@ -929,12 +929,15 @@ const UserAnalyticsPage: React.FC = () => { Preference Data Access

-

- {storeNameMap.get(selectedEntryForDetails.storeId) || - "A store"}{" "} - accessed your preference data. This allows them to - personalize your shopping experience based on your - interests. +

+ {selectedEntryForDetails.storeId + ? (storeNameMap.get( + selectedEntryForDetails.storeId, + ) ?? `ID: ${selectedEntryForDetails.storeId}`) + : "N/A"}

You can manage which stores can access your data in the diff --git a/web/src/pages/UserDashboard/UserDashboard.tsx b/web/src/pages/UserDashboard/UserDashboard.tsx index f8e3fa6..280a060 100644 --- a/web/src/pages/UserDashboard/UserDashboard.tsx +++ b/web/src/pages/UserDashboard/UserDashboard.tsx @@ -517,7 +517,7 @@ export default function UserDashboard() { {entry.dataType === "preference_access" - ? `Preferences accessed by ${storeNameMap.get(entry.storeId) || "Unknown Store"}` + ? `Preferences accessed by ${entry.storeId ? storeNameMap.get(entry.storeId) || "Unknown Store" : "Unknown Store"}` : `${entry.dataType}${entry.storeId ? ` at ${storeNameMap.get(entry.storeId) || "Unknown Store"}` : ""}`}