WIP: support VESC Tool's CSV data format

package.json

@@ -17,22 +17,23 @@
     "ci:bump": "tsx scripts/bump.ts"
   },
   "devDependencies": {
-    "@sveltejs/vite-plugin-svelte": "^4.0.0-next.7",
+    "@sveltejs/vite-plugin-svelte": "^4.0.0",
     "@testing-library/jest-dom": "^6.5.0",
-    "@testing-library/svelte": "^5.2.1",
+    "@testing-library/svelte": "^5.2.4",
     "@tsconfig/svelte": "^5.0.4",
     "@types/leaflet": "^1.9.12",
     "@types/papaparse": "^5.3.14",
     "@types/semver": "^7.5.8",
     "autoprefixer": "^10.4.20",
+    "blob-polyfill": "^9.0.20240710",
     "concurrently": "^9.0.1",
     "jsdom": "^25.0.0",
     "postcss": "^8.4.45",
     "prettier": "^3.3.3",
     "prettier-plugin-svelte": "^3.2.6",
     "semver": "^7.6.3",
-    "svelte": "^5.0.0-next.243",
-    "svelte-check": "^3.8.6",
+    "svelte": "^5.1.3",
+    "svelte-check": "^4.0.5",
     "tailwindcss": "^3.4.11",
     "tslib": "^2.6.3",
     "tsx": "^4.19.1",

src/components/Map.svelte

@@ -53,11 +53,9 @@
 
   $effect(() => {
     if (map) {
-      untrack(() => {
-        if (riderMarker) {
-          riderMarker.remove();
-        }
-      });
+      if (riderMarker) {
+        riderMarker.remove();
+      }
 
       const latLng = gpsPoints[selectedRowIndex];
       if (latLng) {
@@ -81,20 +79,20 @@
   });
 
   $effect(() => {
-    untrack(() => {
+    // ensure this is updated each time the visible rows change
+    visibleRows;
+
+    basePolyline = getBaseLine(gpsPoints, gpsGaps, hiddenRideSegments).addTo(map!);
+    travelledPolyline = getTravelledLine(gpsPoints, gpsGaps, selectedRowIndex, hiddenRideSegments).addTo(map!);
+
+    return () => {
       if (basePolyline) {
         basePolyline.remove();
       }
       if (travelledPolyline) {
         travelledPolyline.remove();
       }
-    });
-
-    // ensure this is updated each time the visible rows change
-    visibleRows;
-
-    basePolyline = getBaseLine(gpsPoints, gpsGaps, hiddenRideSegments).addTo(map!);
-    travelledPolyline = getTravelledLine(gpsPoints, gpsGaps, selectedRowIndex, hiddenRideSegments).addTo(map!);
+    };
   });
 
   function setVisibleIndices() {

src/lib/misc.ts

@@ -26,3 +26,24 @@ export const formatFloat = (n: number | undefined, allowInt = false) => {
   if (allowInt && Number.isInteger(n)) return n.toString();
   return n.toFixed(1);
 };
+
+const loggedHeaders = new Set<string>();
+export const createHeaderTransformer = (mapper: Record<string, string | undefined>) => (header: string) => {
+  const key = mapper[header];
+  if (key === undefined && !loggedHeaders.has(header)) {
+    console.warn('Unknown header found in CSV file', { header });
+    loggedHeaders.add(header);
+  }
+
+  return key ?? header;
+};
+
+export const parseFloatValue = (input: string): number => {
+  const float = parseFloat(input);
+  if (Number.isNaN(float)) {
+    console.warn(`Failed to parse CSV! Expected a number, but got: '${input}'`);
+    return 0;
+  }
+
+  return float;
+};

src/lib/parse/float-control.test.ts

@@ -13,48 +13,48 @@ describe(parseFloatControlCsv.name, () => {
   beforeEach(() => vi.restoreAllMocks());
 
   test('metric', async () => {
-    const { units, csv } = await parseFloatControlCsv(csvMetric);
+    const { units, data, error } = await parseFloatControlCsv(csvMetric);
     expect(units).toBe('metric');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([defaultFixture]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([defaultFixture]);
   });
 
   test('metric with bms', async () => {
-    const { units, csv } = await parseFloatControlCsv(csvMetricWithBms);
+    const { units, data, error } = await parseFloatControlCsv(csvMetricWithBms);
     expect(units).toBe('metric');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([{ ...defaultFixture, ...defaultBms }]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([{ ...defaultFixture, ...defaultBms }]);
   });
 
   test('imperial', async () => {
-    const { units, csv } = await parseFloatControlCsv(csvImperial);
+    const { units, data, error } = await parseFloatControlCsv(csvImperial);
     expect(units).toBe('imperial');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([defaultFixture]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([defaultFixture]);
   });
 
   test('imperial with bms', async () => {
-    const { units, csv } = await parseFloatControlCsv(csvImperialWithBms);
+    const { units, data, error } = await parseFloatControlCsv(csvImperialWithBms);
     expect(units).toBe('imperial');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([{ ...defaultFixture, ...defaultBms }]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([{ ...defaultFixture, ...defaultBms }]);
   });
 
   test('with unknown state', async () => {
     const warnSpy = vi.spyOn(console, 'warn').mockReturnValue();
-    const { units, csv } = await parseFloatControlCsv(csvWithUnknownState);
+    const { units, data, error } = await parseFloatControlCsv(csvWithUnknownState);
     expect(units).toBe('metric');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([{ ...defaultFixture, state: 'some_new_state' }]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([{ ...defaultFixture, state: 'some_new_state' }]);
     expect(warnSpy).toHaveBeenCalledWith("Unknown state: 'SOME_NEW_STATE'");
   });
 
   test('with bad time', async () => {
     const warnSpy = vi.spyOn(console, 'warn').mockReturnValue();
-    const { units, csv } = await parseFloatControlCsv(csvWithBadtime);
+    const { units, data, error } = await parseFloatControlCsv(csvWithBadtime);
     expect(units).toBe('metric');
-    expect(csv.errors).toEqual([]);
-    expect(csv.data).toEqual([{ ...defaultFixture, time: 0 }]);
+    expect(error).toBeUndefined();
+    expect(data).toEqual([{ ...defaultFixture, time: 0 }]);
     expect(warnSpy).toHaveBeenCalledWith("Failed to parse CSV! Expected a number, but got: 'I am not a number'");
   });
 });

src/lib/parse/float-control.ts

@@ -1,24 +1,11 @@
-import csv, { type ParseResult } from 'papaparse';
-import { floatControlKeyMap, FloatControlRawHeader } from './float-control.types';
+import csv from 'papaparse';
+import { floatControlToRowMap, FloatControlRawHeader } from './float-control.types';
 import demoCsv from '../../assets/demo.csv?raw';
-import { attachIndex } from '../misc';
-import { RowKey, State, type Row, type RowWithIndex, Units } from './types';
+import { attachIndex, createHeaderTransformer, parseFloatValue } from '../misc';
+import { RowKey, State, type Row, type RowWithIndex, Units, DataSource } from './types';
+import { ParseError, type ParseResult } from './index';
 
-const transformHeader = (header: string) => {
-  const key = floatControlKeyMap[header as FloatControlRawHeader];
-  if (!key) console.warn('Unknown header found in CSV file', { header });
-  return key ?? header;
-};
-
-const parseFloatValue = (input: string): number => {
-  const float = parseFloat(input);
-  if (Number.isNaN(float)) {
-    console.warn(`Failed to parse CSV! Expected a number, but got: '${input}'`);
-    return 0;
-  }
-
-  return float;
-};
+const transformHeader = createHeaderTransformer(floatControlToRowMap);
 
 const transform = <C extends RowKey>(value: string, column: C): Row[C] => {
   switch (column) {
@@ -56,12 +43,7 @@ const parseOptions = {
   transform,
 };
 
-export interface FloatControlData {
-  csv: ParseResult<RowWithIndex>;
-  units: Units;
-}
-
-export function parseFloatControlCsv(input: string | File): Promise<FloatControlData> {
+export function parseFloatControlCsv(input: string | File): Promise<ParseResult> {
   let units = Units.Imperial;
   return new Promise((resolve) => {
     csv.parse<Row>(input, {
@@ -74,10 +56,15 @@ export function parseFloatControlCsv(input: string | File): Promise<FloatControl
         return transformHeader(header);
       },
       complete: (results) => {
-        results.data = attachIndex(results.data);
+        const data = attachIndex(results.data);
         resolve({
-          csv: results as ParseResult<RowWithIndex>,
+          source: DataSource.FloatControl,
+          data,
           units,
+          error:
+            results.errors.length > 0
+              ? new ParseError('Failed to parse Float Control CSV!', results.errors)
+              : undefined,
         });
       },
     });

src/lib/parse/float-control.types.ts

@@ -44,7 +44,7 @@ export enum FloatControlRawHeader {
   WhCharged = 'Wh Charged',
 }
 
-export const floatControlKeyMap: Record<FloatControlRawHeader, RowKey> = {
+export const floatControlToRowMap: Record<FloatControlRawHeader, RowKey> = {
   [FloatControlRawHeader.Adc1]: RowKey.Adc1,
   [FloatControlRawHeader.Adc2]: RowKey.Adc2,
   [FloatControlRawHeader.Ah]: RowKey.Ah,

src/lib/parse/index.ts

@@ -3,6 +3,7 @@ import * as fflate from 'fflate';
 import { parseFloatControlCsv } from './float-control';
 import { parseFloatyJson } from './floaty';
 import { DataSource, Units, type RowWithIndex } from './types';
+import { parseVescToolCsv } from './vesc';
 
 export class ParseError extends Error {
   constructor(
@@ -21,9 +22,9 @@ export interface ParseResult {
 }
 
 export enum SupportedMimeTypes {
-  /** Float Control's CSV */
+  /** VESC Tool or Float Control CSV */
   Csv = 'text/csv',
-  /** Float Control's Zipped CSV */
+  /** VESC Tool or Float Control Zipped CSV */
   Zip1 = 'application/zip',
   Zip2 = 'application/x-zip-compressed',
   /** Floaty's JSON */
@@ -33,6 +34,21 @@ export enum SupportedMimeTypes {
 export const supportedMimeTypes = Object.values<string>(SupportedMimeTypes);
 export const supportedMimeTypeString = supportedMimeTypes.join(',');
 
+async function parseCsv(input: string | File): Promise<ParseResult> {
+  if (typeof input !== 'string') {
+    const first100Bytes = await input.slice(0, 100).text();
+    const semiColonCount = first100Bytes.split(';').length - 1;
+
+    // VESC Tool delimits by semi-colon, so we use that to detect if it's CSV
+    // generated by VESC Tool or Float Control
+    if (semiColonCount > 1) {
+      return parseVescToolCsv(input);
+    }
+  }
+
+  return await parseFloatControlCsv(input);
+}
+
 export async function parse(file: File): Promise<ParseResult> {
   const lowerName = file.name.toLowerCase();
   if (file.type === SupportedMimeTypes.Zip1 || file.type === SupportedMimeTypes.Zip2 || lowerName.endsWith('.zip')) {
@@ -48,29 +64,11 @@ export async function parse(file: File): Promise<ParseResult> {
     }
 
     const unzippedBytes = fileMap[fileList[0]!]!;
-    const parsed = await parseFloatControlCsv(new TextDecoder().decode(unzippedBytes));
-    return {
-      source: DataSource.FloatControl,
-      data: parsed.csv.data,
-      units: parsed.units,
-      error:
-        parsed.csv.errors.length > 0
-          ? new ParseError('Failed to parse Float Control CSV properly!', parsed.csv.errors)
-          : undefined,
-    };
+    return await parseCsv(new TextDecoder().decode(unzippedBytes));
   }
 
   if (file.type === SupportedMimeTypes.Csv || lowerName.endsWith('.csv')) {
-    const parsed = await parseFloatControlCsv(file);
-    return {
-      source: DataSource.FloatControl,
-      data: parsed.csv.data,
-      units: parsed.units,
-      error:
-        parsed.csv.errors.length > 0
-          ? new ParseError('Failed to parse Float Control CSV properly!', parsed.csv.errors)
-          : undefined,
-    };
+    return await parseCsv(file);
   }
 
   if (file.type === SupportedMimeTypes.Json || lowerName.endsWith('.json')) {

src/lib/parse/types.ts

@@ -2,6 +2,7 @@ export enum DataSource {
   None = 'none',
   FloatControl = 'float_control',
   Floaty = 'floaty',
+  VescTool = 'vesc_tool',
 }
 
 export enum Units {
@@ -63,6 +64,7 @@ export const empty: RowWithIndex = {
   wh: 0,
 };
 
+// TODO: make keyof
 export enum RowKey {
   Adc1 = 'adc1',
   Adc2 = 'adc2',