diff --git a/package.json b/package.json index 85b5c4551..120618cd7 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "flatfilers/*", "plugins/*", "support/*", - "utils/*" + "utils/*", + "validators/*" ], "scripts": { "clean": "find ./ '(' -name 'node_modules' -o -name 'dist' -o -name '.turbo' -o -name '.parcel-cache' ')' -type d -exec rm -rf {} +", diff --git a/validators/StravaGPXFetcher/README.MD b/validators/StravaGPXFetcher/README.MD new file mode 100644 index 000000000..a9a180091 --- /dev/null +++ b/validators/StravaGPXFetcher/README.MD @@ -0,0 +1,60 @@ +# Strava GPX Fetcher Plugin for Flatfile + +This Flatfile Listener plugin fetches GPX data from Strava, parses it, and converts it to a tabular format suitable for Flatfile. It includes authentication with the Strava API, data fetching, parsing, and caching to optimize performance. + +## Features + +- Authenticates with Strava API +- Fetches GPX data for specified Strava activities +- Parses GPX data and converts it to tabular format +- Caches fetched data to reduce API calls +- Integrates seamlessly with Flatfile Listener +- Handles error logging and information logging + +## Installation + +Install the plugin using npm: + +```bash +npm install @flatfile/plugin-strava-gpx-fetcher +``` + +## Example Usage + +```javascript +import { FlatfileListener } from '@flatfile/listener'; +import { stravaGPXFetcher } from '@flatfile/plugin-strava-gpx-fetcher'; + +const listener = new FlatfileListener(); + +const config = { + clientId: 'your_strava_client_id', + clientSecret: 'your_strava_client_secret', + redirectUri: 'your_redirect_uri' +}; + +listener.use(stravaGPXFetcher(config)); + +// ... rest of your Flatfile setup +``` + +## Configuration + +The plugin requires the following configuration: + +- `clientId`: Your Strava API client ID +- `clientSecret`: Your Strava API client secret +- `redirectUri`: The redirect URI set in your Strava API application + +## Behavior + +1. The plugin listens for the 'job:ready' event. +2. When triggered, it sets up a listener for the 'strava:fetch-gpx' event. +3. On 'strava:fetch-gpx', it: + - Checks the cache for existing GPX data + - If not cached, authenticates with Strava and fetches the GPX data + - Parses the GPX data and converts it to tabular format + - Inserts the data into the specified Flatfile sheet +4. The plugin handles errors and logs information throughout the process. + +Note: This plugin requires an authorization code obtained through the OAuth flow with Strava. Ensure you have set up the OAuth flow in your application before using this plugin. \ No newline at end of file diff --git a/validators/StravaGPXFetcher/metadata.json b/validators/StravaGPXFetcher/metadata.json new file mode 100644 index 000000000..12d2ef8f5 --- /dev/null +++ b/validators/StravaGPXFetcher/metadata.json @@ -0,0 +1,105 @@ +{ + "timestamp": "2024-09-24T08-20-42-867Z", + "task": "Create a Strava GPX Fetcher Flatfile Listener plugin:\n - Implement a custom action to fetch GPX data from Strava activities\n - Integrate with Strava API for authentication and data retrieval\n - Allow users to specify Strava activity IDs or date ranges for fetching\n - Download and parse GPX files from specified Strava activities\n - Extract key information such as route, elevation, and time data\n - Convert the GPX data into a tabular format suitable for Flatfile\n - Implement error handling for API rate limits and authentication issues\n - Provide options for data filtering and aggregation (e.g., by activity type)\n - Calculate and include additional metrics like average speed and moving time\n - Allow batch processing of multiple Strava activities\n - Implement caching mechanism to avoid redundant API calls for previously fetched activities", + "summary": "This code implements a Flatfile Listener plugin for fetching Strava GPX data. It includes authentication with the Strava API, fetching GPX data, parsing it, and converting it to a tabular format suitable for Flatfile. The plugin also includes caching to avoid redundant API calls and utility functions for data processing.", + "steps": [ + [ + "Gather information about Flatfile Listeners and the necessary structure for creating a custom plugin.\n", + "#E1", + "PineconeAssistant", + "What is the basic structure of a Flatfile Listener plugin and how to create a custom action?", + "Plan: Gather information about Flatfile Listeners and the necessary structure for creating a custom plugin.\n#E1 = PineconeAssistant[What is the basic structure of a Flatfile Listener plugin and how to create a custom action?]" + ], + [ + "Research the Strava API authentication process and required endpoints for fetching GPX data.\n", + "#E2", + "Google", + "Strava API authentication and GPX data retrieval endpoints", + "Plan: Research the Strava API authentication process and required endpoints for fetching GPX data.\n#E2 = Google[Strava API authentication and GPX data retrieval endpoints]" + ], + [ + "Create the basic structure of the Strava GPX Fetcher Flatfile Listener plugin.\n", + "#E3", + "LLM", + "Using the information from #E1 and #E2, create the basic structure of a Flatfile Listener plugin for Strava GPX fetching, including the necessary imports and a custom action for fetching GPX data", + "Plan: Create the basic structure of the Strava GPX Fetcher Flatfile Listener plugin.\n#E3 = LLM[Using the information from #E1 and #E2, create the basic structure of a Flatfile Listener plugin for Strava GPX fetching, including the necessary imports and a custom action for fetching GPX data]" + ], + [ + "Implement the Strava API authentication function.\n", + "#E4", + "LLM", + "Based on #E2 and #E3, implement a function to handle Strava API authentication", + "Plan: Implement the Strava API authentication function.\n#E4 = LLM[Based on #E2 and #E3, implement a function to handle Strava API authentication]" + ], + [ + "Create functions to fetch GPX data from Strava activities based on activity IDs or date ranges.\n", + "#E5", + "LLM", + "Implement functions to fetch GPX data from Strava API using activity IDs or date ranges, incorporating error handling for API rate limits and authentication issues", + "Plan: Create functions to fetch GPX data from Strava activities based on activity IDs or date ranges.\n#E5 = LLM[Implement functions to fetch GPX data from Strava API using activity IDs or date ranges, incorporating error handling for API rate limits and authentication issues]" + ], + [ + "Implement GPX parsing and data extraction functions.\n", + "#E6", + "LLM", + "Create functions to parse GPX files and extract key information such as route, elevation, and time data", + "Plan: Implement GPX parsing and data extraction functions.\n#E6 = LLM[Create functions to parse GPX files and extract key information such as route, elevation, and time data]" + ], + [ + "Develop functions to convert GPX data into a tabular format suitable for Flatfile.\n", + "#E7", + "LLM", + "Implement functions to convert parsed GPX data into a tabular format compatible with Flatfile", + "Plan: Develop functions to convert GPX data into a tabular format suitable for Flatfile.\n#E7 = LLM[Implement functions to convert parsed GPX data into a tabular format compatible with Flatfile]" + ], + [ + "Create utility functions for data filtering, aggregation, and additional metric calculations.\n", + "#E8", + "LLM", + "Develop utility functions for filtering data by activity type, aggregating multiple activities, and calculating metrics like average speed and moving time", + "Plan: Create utility functions for data filtering, aggregation, and additional metric calculations.\n#E8 = LLM[Develop utility functions for filtering data by activity type, aggregating multiple activities, and calculating metrics like average speed and moving time]" + ], + [ + "Implement a caching mechanism to avoid redundant API calls.\n", + "#E9", + "LLM", + "Create a caching system to store previously fetched activity data and avoid unnecessary API calls", + "Plan: Implement a caching mechanism to avoid redundant API calls.\n#E9 = LLM[Create a caching system to store previously fetched activity data and avoid unnecessary API calls]" + ], + [ + "Integrate all components into the main Flatfile Listener plugin structure.\n", + "#E10", + "LLM", + "Combine all the implemented functions and components from #E3 to #E9 into a cohesive Flatfile Listener plugin structure", + "Plan: Integrate all components into the main Flatfile Listener plugin structure.\n#E10 = LLM[Combine all the implemented functions and components from #E3 to #E9 into a cohesive Flatfile Listener plugin structure]" + ], + [ + "Validate the final code, ensure correct usage of Flatfile plugins and utils, and verify Event Topic subscriptions.\n", + "#E11", + "PineconeAssistant", + "Validate the following Flatfile Listener plugin code, check for correct usage of Flatfile plugins and utils, and verify that the Event Topic subscriptions are valid: #E10", + "Plan: Validate the final code, ensure correct usage of Flatfile plugins and utils, and verify Event Topic subscriptions.\n#E11 = PineconeAssistant[Validate the following Flatfile Listener plugin code, check for correct usage of Flatfile plugins and utils, and verify that the Event Topic subscriptions are valid: #E10]" + ], + [ + "Make any necessary adjustments based on the validation results.\n", + "#E12", + "LLM", + "Based on the validation results in #E11, make any required adjustments to the Flatfile Listener plugin code", + "Plan: Make any necessary adjustments based on the validation results.\n#E12 = LLM[Based on the validation results in #E11, make any required adjustments to the Flatfile Listener plugin code]" + ], + [ + "Perform a final check for unused imports and ensure all parameters are correct.\n", + "#E13", + "LLM", + "Review the final Flatfile Listener plugin code from #E12, remove any unused imports, and verify that all parameters are correct for the used plugins and functions", + "Plan: Perform a final check for unused imports and ensure all parameters are correct.\n#E13 = LLM[Review the final Flatfile Listener plugin code from #E12, remove any unused imports, and verify that all parameters are correct for the used plugins and functions]" + ] + ], + "metrics": { + "tokens": { + "plan": 3959, + "state": 7023, + "total": 10982 + } + } +} \ No newline at end of file diff --git a/validators/StravaGPXFetcher/package.json b/validators/StravaGPXFetcher/package.json new file mode 100644 index 000000000..ac33d10db --- /dev/null +++ b/validators/StravaGPXFetcher/package.json @@ -0,0 +1,72 @@ +{ + "name": "@flatfile/plugin-strava-gpx-fetcher", + "version": "1.0.0", + "description": "A Flatfile plugin for fetching and processing Strava GPX data", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "browser": { + "./dist/index.js": "./dist/index.browser.js", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "browser": { + "require": "./dist/index.browser.js", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + }, + "keywords": [ + "flatfile", + "plugin", + "strava", + "gpx", + "fetcher", + "flatfile-plugins", + "category-transform" + ], + "author": "Your Name", + "license": "MIT", + "dependencies": { + "@flatfile/listener": "^1.0.5", + "@flatfile/util-common": "^1.3.8", + "@flatfile/api": "^1.9.15", + "axios": "^1.7.7", + "fast-xml-parser": "^4.5.0" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "typescript": "^5.6.2", + "@types/node": "^22.6.1" + }, + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/strava-gpx-fetcher" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ] +} \ No newline at end of file diff --git a/validators/StravaGPXFetcher/rollup.config.mjs b/validators/StravaGPXFetcher/rollup.config.mjs new file mode 100644 index 000000000..0cd860e4f --- /dev/null +++ b/validators/StravaGPXFetcher/rollup.config.mjs @@ -0,0 +1,58 @@ +import { buildConfig } from '@flatfile/rollup-config'; +import typescript from '@rollup/plugin-typescript'; +import nodeResolve from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; +import json from '@rollup/plugin-json'; + +const umdExternals = [ + '@flatfile/api', + '@flatfile/listener', + '@flatfile/util-common', + 'axios', + 'fast-xml-parser' +]; + +const config = buildConfig({ + includeUmd: true, + umdConfig: { name: 'StravaGPXFetcher', external: umdExternals }, + external: [ + ...umdExternals, + 'crypto', + 'util', + 'stream', + 'zlib', + 'http', + 'https', + 'url', + 'net' + ] +}); + +// Add TypeScript configuration +config.forEach(conf => { + if (conf.plugins) { + conf.plugins.push( + typescript({ + tsconfig: './tsconfig.json', + declaration: true, + declarationDir: './dist/types/' + }) + ); + } +}); + +// Ensure proper handling of dependencies +config.forEach(conf => { + if (conf.plugins) { + conf.plugins.unshift( + nodeResolve({ + preferBuiltins: true, + browser: conf.output.format === 'umd' + }), + commonjs(), + json() + ); + } +}); + +export default config; \ No newline at end of file diff --git a/validators/StravaGPXFetcher/src/index.ts b/validators/StravaGPXFetcher/src/index.ts new file mode 100644 index 000000000..c69a75d58 --- /dev/null +++ b/validators/StravaGPXFetcher/src/index.ts @@ -0,0 +1,122 @@ +import { FlatfileListener, FlatfileEvent } from '@flatfile/listener' +import { logInfo, logError } from '@flatfile/util-common' +import api from '@flatfile/api' +import axios from 'axios' +import { XMLParser } from 'fast-xml-parser' + +interface StravaGPXFetcherConfig { + clientId: string + clientSecret: string + redirectUri: string +} + +interface CacheItem { + value: any + timestamp: number +} + +class Cache { + private cache: Map = new Map() + private ttl: number + + constructor(ttl: number = 3600000) { + // TTL in milliseconds, default 1 hour + this.ttl = ttl + } + + get(key: string): any { + const item = this.cache.get(key) + if (item && Date.now() - item.timestamp < this.ttl) { + return item.value + } + this.cache.delete(key) + return null + } + + set(key: string, value: any): void { + this.cache.set(key, { value, timestamp: Date.now() }) + } +} + +async function authenticateStrava( + config: StravaGPXFetcherConfig +): Promise { + const tokenResponse = await axios.post('https://www.strava.com/oauth/token', { + client_id: config.clientId, + client_secret: config.clientSecret, + grant_type: 'authorization_code', + code: '', // This should be obtained through OAuth flow + }) + return tokenResponse.data.access_token +} + +async function fetchGPXData( + accessToken: string, + activityId: string +): Promise { + const response = await axios.get( + `https://www.strava.com/api/v3/activities/${activityId}/streams?keys=latlng,altitude,time&key_by_type=true`, + { + headers: { Authorization: `Bearer ${accessToken}` }, + } + ) + return response.data +} + +function parseGPX(gpxData: string): any { + const parser = new XMLParser() + return parser.parse(gpxData) +} + +function convertToTabularFormat(parsedGPX: any): any[] { + // This is a simplified conversion. Adjust based on your specific GPX structure and requirements. + const trackPoints = parsedGPX.gpx.trk.trkseg.trkpt + return trackPoints.map((point: any) => ({ + latitude: point['@_lat'], + longitude: point['@_lon'], + elevation: point.ele, + time: point.time, + })) +} + +export function stravaGPXFetcher(config: StravaGPXFetcherConfig) { + const cache = new Cache() + + return (listener: FlatfileListener) => { + listener.use((handler) => { + handler.on('job:ready', async (event: FlatfileEvent) => { + try { + logInfo('strava-gpx-fetcher', 'Job ready event received') + + handler.on('strava:fetch-gpx', async (event: FlatfileEvent) => { + const { activityId } = event.payload + + let gpxData = cache.get(activityId) + if (!gpxData) { + const accessToken = await authenticateStrava(config) + gpxData = await fetchGPXData(accessToken, activityId) + cache.set(activityId, gpxData) + } + + const parsedGPX = parseGPX(gpxData) + const tabularData = convertToTabularFormat(parsedGPX) + + // Assuming we're working with a specific sheet in the Flatfile space + const sheetId = event.context.sheetId + await api.records.insert(sheetId, tabularData) + + logInfo( + 'strava-gpx-fetcher', + `GPX data fetched and inserted for activity ${activityId}` + ) + }) + } catch (error) { + logError( + 'strava-gpx-fetcher', + `Error processing event: ${error.message}` + ) + } + }) + }) + } +}