feature/handle-headers

.github/workflows/build-test.yml

@@ -0,0 +1,30 @@
+# This workflow will build a .NET project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net
+
+name: .NET
+
+on:
+  push:
+    branches: [ "feature/*" ]
+
+env:
+  VERSION: 0.0.0-local
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read   # default is read; fine for checkout/build/test
+      actions: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.0.x
+      - name: Restore
+        run: dotnet restore
+      - name: Build
+        run: dotnet build --no-restore -c Release
+      - name: Test
+        run: dotnet test -c Release --no-build --verbosity normal

.github/workflows/build.yml → .github/workflows/publish.yml

@@ -64,7 +64,7 @@ jobs:
           fi
           echo "VERSION=$VERSION" >> "$GITHUB_ENV"
       - name: Publish self-contained single file
-        run: dotnet publish src/CsvToOfx.Cli/CsvToOfx.Cli.csproj -c Release -r ${{ matrix.rid }} --self-contained true /p:PublishSingleFile=true /p:IncludeAllContentForSelfExtract=true /p:Version=${{ env.VERSION }}
+        run: dotnet publish src/CsvToOfx.Cli/CsvToOfx.Cli.csproj -c Release -r ${{ matrix.rid }} --self-contained true --framework net8.0 /p:PublishSingleFile=true /p:IncludeAllContentForSelfExtract=true /p:Version=${{ env.VERSION }}
       - name: Archive artifact
         run: |
           cd src/CsvToOfx.Cli/bin/Release/net8.0/${{ matrix.rid }}/publish

src/CsvToOfx.Core/CsvToOfx.Core.csproj

@@ -9,6 +9,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.1" />
+    <PackageReference Include="CsvHelper" Version="30.0.1" />
   </ItemGroup>
 
   <ItemGroup>

src/CsvToOfx.Core/Models/CanonicalAction.cs

@@ -1,2 +1,10 @@
 namespace CsvToOfx.Core.Models;
-public enum CanonicalAction { Income, BuyStock, SellStock, StockSplit, MiscExpense, CashTransfer }
+public enum CanonicalAction { 
+    Income, 
+    BuyStock, 
+    SellStock, 
+    Reinvest, 
+    StockSplit, 
+    CashTransfer, 
+    MiscExpense 
+}

src/CsvToOfx.Core/Models/CanonicalField.cs

@@ -0,0 +1,23 @@
+namespace CsvToOfx.Core.Models;
+
+public enum CanonicalField
+{
+    TradeDate,
+    Action,
+    Symbol,
+    Description,
+    Type,
+    ExchangeQuantity,
+    ExchangeCurrency,
+    Currency,
+    Price,
+    Quantity,
+    ExchangeRate,
+    Commission,
+    Fees,
+    AccruedInterest,
+    Amount,
+    CashBalance,
+    SettlementDate
+}
+

src/CsvToOfx.Core/Models/HeaderMap.cs

@@ -0,0 +1,8 @@
+using System.Collections.Generic;
+
+namespace CsvToOfx.Core.Models;
+
+public sealed record HeaderMap(
+    string Name,
+    IReadOnlyDictionary<string, CanonicalField> Columns
+);

src/CsvToOfx.Core/Parsing/ActionResolverAdapter.cs

@@ -0,0 +1,19 @@
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing;
+
+public sealed class ActionResolverAdapter : IActionResolver
+{
+    public CanonicalAction Resolve(string? actionText)
+    {
+        var a = (actionText ?? string.Empty).Trim().ToLowerInvariant();
+        if (a.Contains("transfer") || a.Contains("transferred")) return CanonicalAction.CashTransfer;
+        if (a.Contains("reverse split") || a.Contains("stock split") || a.Contains("split r/s") || a.Contains("r/s to") || a.Contains("r/s from")) return CanonicalAction.StockSplit;
+        if (a.Contains("dividend") || a.Contains("interest") || a.Contains("return of capital") || a.Contains("in lieu")) return CanonicalAction.Income;
+        if (a.Contains("bought") || a.Contains("buy") || a.Contains("purchase") || a.Contains("reinvestment")) return CanonicalAction.BuyStock;
+        if (a.Contains("sold") || a.Contains("sell")) return CanonicalAction.SellStock;
+        if (a.Contains("fee charged") || a.Contains("adr fee") || a.Contains("adr pass-through fee")
+            || a.Contains("foreign tax paid") || a.Contains("withholding tax") || a.Contains("tax withheld")) return CanonicalAction.MiscExpense;
+        return CanonicalAction.CashTransfer;
+    }
+}

src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityIraHeaderMap.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing.HeaderMaps;
+
+public static class FidelityIraHeaderMap
+{
+    public static HeaderMap Instance { get; } = new HeaderMap(
+        "Fidelity-IRA",
+        new Dictionary<string, CanonicalField>(StringComparer.OrdinalIgnoreCase)
+        {
+            ["Run Date"] = CanonicalField.TradeDate,
+            ["Action"] = CanonicalField.Action,
+            ["Symbol"] = CanonicalField.Symbol,
+            ["Description"] = CanonicalField.Description,
+            ["Type"] = CanonicalField.Type,
+            ["Price ($)"] = CanonicalField.Price,
+            ["Quantity"] = CanonicalField.Quantity,
+            ["Commission ($)"] = CanonicalField.Commission,
+            ["Fees ($)"] = CanonicalField.Fees,
+            ["Accrued Interest ($)"] = CanonicalField.AccruedInterest,
+            ["Amount ($)"] = CanonicalField.Amount,
+            ["Cash Balance ($)"] = CanonicalField.CashBalance,
+            ["Settlement Date"] = CanonicalField.SettlementDate
+        });
+}
+

src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityTradingHeaderMap.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing.HeaderMaps;
+
+public static class FidelityTradingHeaderMap
+{
+    public static HeaderMap Instance { get; } = new HeaderMap(
+        "Fidelity-Trading",
+        new Dictionary<string, CanonicalField>(StringComparer.OrdinalIgnoreCase)
+        {
+            ["Run Date"] = CanonicalField.TradeDate,
+            ["Action"] = CanonicalField.Action,
+            ["Symbol"] = CanonicalField.Symbol,
+            ["Description"] = CanonicalField.Description,
+            ["Type"] = CanonicalField.Type,
+            ["Exchange Quantity"] = CanonicalField.ExchangeQuantity,
+            ["Exchange Currency"] = CanonicalField.ExchangeCurrency,
+            ["Currency"] = CanonicalField.Currency,
+            ["Price"] = CanonicalField.Price,
+            ["Quantity"] = CanonicalField.Quantity,
+            ["Exchange Rate"] = CanonicalField.ExchangeRate,
+            ["Commission"] = CanonicalField.Commission,
+            ["Fees"] = CanonicalField.Fees,
+            ["Accrued Interest"] = CanonicalField.AccruedInterest,
+            ["Amount"] = CanonicalField.Amount,
+            ["Cash Balance"] = CanonicalField.CashBalance,
+            ["Settlement Date"] = CanonicalField.SettlementDate
+        });
+}
+

src/CsvToOfx.Core/Parsing/IActionResolver.cs

@@ -0,0 +1,9 @@
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing;
+
+public interface IActionResolver
+{
+    CanonicalAction Resolve(string? actionText);
+}
+

src/CsvToOfx.Core/Parsing/ISecurityResolver.cs

@@ -0,0 +1,10 @@
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing;
+
+public interface ISecurityResolver
+{
+    SecurityRef? Resolve(string symbol);
+    SecurityRef? ResolveFromRow(IDictionary<string, string?> row);
+}
+

src/CsvToOfx.Core/Parsing/TransactionCsvParser.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using CsvHelper;
+using CsvHelper.Configuration;
+using CsvToOfx.Core.Models;
+
+namespace CsvToOfx.Core.Parsing;
+
+public sealed class TransactionCsvParser
+{
+    private readonly HeaderMap _headerMap;
+    private readonly ISecurityResolver _securityResolver;
+    private readonly IActionResolver _actionResolver;
+
+    public TransactionCsvParser(
+        HeaderMap headerMap,
+        ISecurityResolver securityResolver,
+        IActionResolver actionResolver)
+    {
+        _headerMap = headerMap;
+        _securityResolver = securityResolver;
+        _actionResolver = actionResolver;
+    }
+
+    public IEnumerable<NormalizedTransaction> Parse(TextReader reader)
+    {
+        using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture)
+        {
+            HasHeaderRecord = true,
+            IgnoreBlankLines = true,
+            BadDataFound = null,
+            TrimOptions = TrimOptions.Trim
+        });
+
+        if (!csv.Read() || !csv.ReadHeader())
+            yield break;
+
+        if (csv.HeaderRecord != null)
+        {
+            var indexMap = BuildIndexMap(csv.HeaderRecord);
+            if (indexMap.Count == 0)
+                yield break;
+
+            while (csv.Read())
+            {
+                if (string.IsNullOrWhiteSpace(csv.Parser.RawRecord))
+                    continue;
+
+                DateTime tradeDate = csv.GetField<DateTime>(indexMap[CanonicalField.TradeDate]);
+                var action = _actionResolver.Resolve(csv.GetField(indexMap[CanonicalField.Action]));
+                var security = ResolveSecurity(csv, indexMap);
+                decimal? units = GetOptional<decimal>(csv, indexMap, CanonicalField.Quantity);
+                decimal? price = GetOptional<decimal>(csv, indexMap, CanonicalField.Price);
+                decimal amount = csv.GetField<decimal>(indexMap[CanonicalField.Amount]);
+                string currency = GetOptional<string>(csv, indexMap, CanonicalField.Currency) ?? "USD";
+                string? memo = GetOptional<string>(csv, indexMap, CanonicalField.Description);
+                decimal? fees = GetOptional<decimal>(csv, indexMap, CanonicalField.Fees);
+                string? fitId = null;
+
+                yield return new NormalizedTransaction(
+                    tradeDate,
+                    action,
+                    security,
+                    units,
+                    price,
+                    amount,
+                    currency,
+                    memo,
+                    fees,
+                    fitId);
+            }
+        }
+    }
+
+    private Dictionary<CanonicalField, int> BuildIndexMap(string[] headerRecord)
+    {
+        var indexMap = new Dictionary<CanonicalField, int>();
+        for (int i = 0; i < headerRecord.Length; i++)
+        {
+            var header = headerRecord[i].Trim();
+            if (_headerMap.Columns.TryGetValue(header, out var field))
+                indexMap[field] = i;
+        }
+        return indexMap;
+    }
+
+    private T? GetOptional<T>(CsvReader csv, Dictionary<CanonicalField, int> indexMap, CanonicalField field)
+    {
+        if (!indexMap.TryGetValue(field, out var idx))
+            return default;
+        var raw = csv.GetField(idx);
+        if (string.IsNullOrWhiteSpace(raw))
+            return default;
+        return csv.GetField<T>(idx);
+    }
+
+    private SecurityRef? ResolveSecurity(CsvReader csv, Dictionary<CanonicalField, int> indexMap)
+    {
+        if (!indexMap.TryGetValue(CanonicalField.Symbol, out var idx))
+            return null;
+        var symbol = csv.GetField(idx);
+        if (symbol != null) 
+            return _securityResolver.Resolve(symbol);
+        else
+        {
+            return null;
+        }
+
+    }
+}
+

src/CsvToOfx.Core/Services/SecurityResolver.cs

@@ -1,9 +1,10 @@
 // C#
 using CsvToOfx.Core.Models;
+using CsvToOfx.Core.Parsing;
 
 namespace CsvToOfx.Core.Services
 {
-    public sealed class SecurityResolver
+    public sealed class SecurityResolver : ISecurityResolver
     {
         private readonly bool _preferCusip;
 
@@ -41,6 +42,16 @@ public SecurityResolver(bool preferCusip = true)
             return new SecurityRef(symbol, idType, displayName, tickerField);
         }
 
+        public SecurityRef? Resolve(string symbol)
+        {
+            if (string.IsNullOrWhiteSpace(symbol)) return null;
+            var row = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
+            {
+                ["Symbol"] = symbol
+            };
+            return ResolveFromRow(row);
+        }
+
         private static bool LooksLikeCusip(string value)
         {
             if (value.Length != 9) return false;

src/CsvToOfx.Parsers/Shared/CsvRowReader.cs

@@ -8,7 +8,7 @@ public sealed class CsvRowReader
     private readonly CsvConfiguration _conf = new(CultureInfo.InvariantCulture)
     {
         HasHeaderRecord = true,
-        DetectColumnCountChanges = true,
+        DetectColumnCountChanges = false,
         BadDataFound = null,
         TrimOptions = TrimOptions.Trim
     };
@@ -21,9 +21,16 @@ public sealed class CsvRowReader
         var seenData = false;
         while (csvr.Read())
         {
+            var colCount = csvr.Parser.Count;
+            if (colCount == 0) continue; // skip empty physical rows
+
             var dict = csvr.GetRecord<dynamic>() as IDictionary<string, object>;
             if (dict is null) continue;
 
+            // drop rows that don't match header shape (e.g., disclaimers/footers)
+            if (csvr.HeaderRecord is { Length: > 0 } && colCount < csvr.HeaderRecord.Length)
+                continue;
+
             var converted = dict.ToDictionary(k => k.Key, v => v.Value?.ToString());
             var nonEmpty = converted.Values.Count(v => !string.IsNullOrWhiteSpace(v));
 

tests/CsvToOfx.Core.Tests/CsvToOfx.Core.Tests.csproj

@@ -4,9 +4,11 @@
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
+    <CollectCoverage>true</CollectCoverage>
+    <CoverletOutputFormat>lcov</CoverletOutputFormat>
+    <CoverletOutput>../TestResults/coverage.info</CoverletOutput>
   </PropertyGroup>
 
   <ItemGroup>