From 2cda8e6f5eabb9730dbd38d20e20782f5b6af372 Mon Sep 17 00:00:00 2001 From: Massimo Fazzolari Date: Wed, 11 Feb 2026 16:05:14 +0100 Subject: [PATCH 1/3] feat: add configurable connection charset (lc_ctype) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add charset option to ConnectOptions allowing users to specify the connection character set used in the DPB (lc_ctype parameter). The charset is also propagated to the data reader and writer so that string encoding/decoding matches the connection charset. This is essential for legacy Firebird databases (commonly created with Delphi/IBX) where columns use charset NONE. In these databases, text is stored as raw bytes in the application's encoding (typically WIN1252) without any charset metadata on the columns. With the current hardcoded 'utf8' charset, the driver tells Firebird to communicate in UTF-8, but Firebird does not transliterate charset NONE columns. The raw WIN1252 bytes are then incorrectly decoded as UTF-8, corrupting accented characters (e.g. 'Tournée' becomes 'Tourn�e'). By setting charset: 'WIN1252' in ConnectOptions, Firebird sends the correct bytes and the driver decodes them using the matching Node.js encoding (latin1, which provides 1:1 byte-to-codepoint mapping for single-byte charsets). Changes: - ConnectOptions: add optional charset property - createDpb(): use options.charset instead of hardcoded 'utf8' - mapCharsetToEncoding(): map Firebird charset names to Node.js encodings - AbstractAttachment: store encoding from connection charset - createDataReader(): accept encoding parameter for string decoding - createDataWriter(): accept encoding parameter for string encoding - AttachmentImpl: set encoding on connect using mapCharsetToEncoding() - StatementImpl: pass attachment.encoding to reader/writer Backward compatible: defaults to 'utf8' when charset is not specified. --- .../src/lib/attachment.ts | 3 ++- .../src/lib/statement.ts | 4 ++-- .../src/lib/impl/attachment.ts | 1 + .../src/lib/impl/fb-util.ts | 18 +++++++++++++----- packages/node-firebird-driver/src/lib/index.ts | 3 +++ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/node-firebird-driver-native/src/lib/attachment.ts b/packages/node-firebird-driver-native/src/lib/attachment.ts index ecd5dc8..a263ec4 100644 --- a/packages/node-firebird-driver-native/src/lib/attachment.ts +++ b/packages/node-firebird-driver-native/src/lib/attachment.ts @@ -3,7 +3,7 @@ import { ClientImpl } from './client'; import { StatementImpl } from './statement'; import { TransactionImpl } from './transaction'; import { EventsImpl } from './events'; -import { createDpb } from './fb-util'; +import { createDpb, mapCharsetToEncoding } from './fb-util'; import { Blob, @@ -27,6 +27,7 @@ export class AttachmentImpl extends AbstractAttachment { static async connect(client: ClientImpl, uri: string, options?: ConnectOptions): Promise { const attachment = new AttachmentImpl(client); + attachment.encoding = mapCharsetToEncoding(options?.charset); return await client.statusAction(async (status) => { const dpb = createDpb(options); diff --git a/packages/node-firebird-driver-native/src/lib/statement.ts b/packages/node-firebird-driver-native/src/lib/statement.ts index ce862c3..059e722 100644 --- a/packages/node-firebird-driver-native/src/lib/statement.ts +++ b/packages/node-firebird-driver-native/src/lib/statement.ts @@ -53,12 +53,12 @@ export class StatementImpl extends AbstractStatement { if (statement.inMetadata) { statement.inBuffer = new Uint8Array(statement.inMetadata.getMessageLengthSync(status)); - statement.dataWriter = createDataWriter(createDescriptors(status, statement.inMetadata)); + statement.dataWriter = createDataWriter(createDescriptors(status, statement.inMetadata), attachment.encoding); } if (statement.outMetadata) { statement.outBuffer = new Uint8Array(statement.outMetadata.getMessageLengthSync(status)); - statement.dataReader = createDataReader(createDescriptors(status, statement.outMetadata)); + statement.dataReader = createDataReader(createDescriptors(status, statement.outMetadata), attachment.encoding); } return statement; diff --git a/packages/node-firebird-driver/src/lib/impl/attachment.ts b/packages/node-firebird-driver/src/lib/impl/attachment.ts index fe9b113..f5c0267 100644 --- a/packages/node-firebird-driver/src/lib/impl/attachment.ts +++ b/packages/node-firebird-driver/src/lib/impl/attachment.ts @@ -19,6 +19,7 @@ import { /** AbstractAttachment implementation. */ export abstract class AbstractAttachment implements Attachment { + encoding: BufferEncoding = 'utf8'; events = new Set(); statements = new Set(); transactions = new Set(); diff --git a/packages/node-firebird-driver/src/lib/impl/fb-util.ts b/packages/node-firebird-driver/src/lib/impl/fb-util.ts index 1089262..2e23c11 100644 --- a/packages/node-firebird-driver/src/lib/impl/fb-util.ts +++ b/packages/node-firebird-driver/src/lib/impl/fb-util.ts @@ -132,9 +132,17 @@ export namespace charSets { export const ascii = 2; } +/** Maps a Firebird charset name to a Node.js encoding. */ +export function mapCharsetToEncoding(charset?: string): BufferEncoding { + if (!charset || charset.toUpperCase() === 'UTF8') { + return 'utf8'; + } else { + return 'latin1'; + } +} export function createDpb(options?: ConnectOptions | CreateDatabaseOptions): Buffer { const code = (c: number) => String.fromCharCode(c); - const charSet = 'utf8'; + const charSet = options?.charset ?? 'utf8'; let ret = `${code(dpb.version1)}${code(dpb.lc_ctype)}${code(charSet.length)}${charSet}`; if (!options) { @@ -306,7 +314,7 @@ export type DataReader = (attachment: Attachment, transaction: Transaction, buff export type ItemReader = (attachment: Attachment, transaction: Transaction, buffer: Uint8Array) => Promise; /** Creates a data reader. */ -export function createDataReader(descriptors: Descriptor[]): DataReader { +export function createDataReader(descriptors: Descriptor[], encoding: BufferEncoding = 'utf8'): DataReader { const mappers = new Array(descriptors.length); for (let i = 0; i < descriptors.length; ++i) { @@ -328,7 +336,7 @@ export function createDataReader(descriptors: Descriptor[]): DataReader { case sqlTypes.SQL_VARYING: { //// TODO: none, octets const varLength = dataView.getUint16(descriptor.offset, littleEndian); - const decoder = new stringDecoder.StringDecoder('utf8'); + const decoder = new stringDecoder.StringDecoder(encoding); const buf = Buffer.from(buffer.buffer, descriptor.offset + 2, varLength); return decoder.end(buf); } @@ -503,7 +511,7 @@ export type ItemWriter = ( ) => Promise; /** Creates a data writer. */ -export function createDataWriter(descriptors: Descriptor[]): DataWriter { +export function createDataWriter(descriptors: Descriptor[], encoding: BufferEncoding = 'utf8'): DataWriter { const mappers = new Array(descriptors.length); for (let i = 0; i < descriptors.length; ++i) { @@ -529,7 +537,7 @@ export function createDataWriter(descriptors: Descriptor[]): DataWriter { case sqlTypes.SQL_VARYING: { //// TODO: none, octets const str = value as string; - const strBuffer = Buffer.from(str); + const strBuffer = Buffer.from(str, encoding); const bytesArray = Uint8Array.from(strBuffer); diff --git a/packages/node-firebird-driver/src/lib/index.ts b/packages/node-firebird-driver/src/lib/index.ts index 5a3da33..28ac7da 100644 --- a/packages/node-firebird-driver/src/lib/index.ts +++ b/packages/node-firebird-driver/src/lib/index.ts @@ -45,6 +45,9 @@ export interface ConnectOptions { /** User role. */ role?: string; + /** Connection charset (lc_ctype). Defaults to 'utf8'. */ + charset?: string; + /** Set database read/write mode. */ setDatabaseReadWriteMode?: DatabaseReadWriteMode; } From 4c68a081deecbb907314800ca75e35d45c5949bf Mon Sep 17 00:00:00 2001 From: Massimo Fazzolari Date: Thu, 12 Feb 2026 17:28:44 +0100 Subject: [PATCH 2/3] feat: configurable connection charset via CharsetCodec Use TextDecoder for reading and a reverse lookup table for writing, which correctly supports all single-byte Firebird charsets (WIN1252, WIN1251, WIN1250, ISO8859_x, etc.) without external dependencies. Replaces the previous latin1-based approach as suggested in review. --- .gitignore | 1 + .../src/lib/attachment.ts | 7 +- .../src/lib/statement.ts | 10 +- .../src/lib/impl/attachment.ts | 1 - .../src/lib/impl/fb-util.ts | 102 +++++++++++++++--- 5 files changed, 104 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index bafa443..b6f1a3b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ lerna-debug.log yarn-error.log tsconfig.tsbuildinfo .env +*.tgz diff --git a/packages/node-firebird-driver-native/src/lib/attachment.ts b/packages/node-firebird-driver-native/src/lib/attachment.ts index a263ec4..b5a4cc3 100644 --- a/packages/node-firebird-driver-native/src/lib/attachment.ts +++ b/packages/node-firebird-driver-native/src/lib/attachment.ts @@ -3,7 +3,7 @@ import { ClientImpl } from './client'; import { StatementImpl } from './statement'; import { TransactionImpl } from './transaction'; import { EventsImpl } from './events'; -import { createDpb, mapCharsetToEncoding } from './fb-util'; +import { createDpb, createCharsetCodec, CharsetCodec } from './fb-util'; import { Blob, @@ -25,9 +25,11 @@ export class AttachmentImpl extends AbstractAttachment { attachmentHandle?: fb.Attachment; + charsetCodec: CharsetCodec; + static async connect(client: ClientImpl, uri: string, options?: ConnectOptions): Promise { const attachment = new AttachmentImpl(client); - attachment.encoding = mapCharsetToEncoding(options?.charset); + attachment.charsetCodec = createCharsetCodec(options?.charset); return await client.statusAction(async (status) => { const dpb = createDpb(options); @@ -42,6 +44,7 @@ export class AttachmentImpl extends AbstractAttachment { options?: CreateDatabaseOptions, ): Promise { const attachment = new AttachmentImpl(client); + attachment.charsetCodec = createCharsetCodec(options?.charset); return await client.statusAction(async (status) => { const dpb = createDpb(options); diff --git a/packages/node-firebird-driver-native/src/lib/statement.ts b/packages/node-firebird-driver-native/src/lib/statement.ts index 059e722..81d393c 100644 --- a/packages/node-firebird-driver-native/src/lib/statement.ts +++ b/packages/node-firebird-driver-native/src/lib/statement.ts @@ -53,12 +53,18 @@ export class StatementImpl extends AbstractStatement { if (statement.inMetadata) { statement.inBuffer = new Uint8Array(statement.inMetadata.getMessageLengthSync(status)); - statement.dataWriter = createDataWriter(createDescriptors(status, statement.inMetadata), attachment.encoding); + statement.dataWriter = createDataWriter( + createDescriptors(status, statement.inMetadata), + attachment.charsetCodec, + ); } if (statement.outMetadata) { statement.outBuffer = new Uint8Array(statement.outMetadata.getMessageLengthSync(status)); - statement.dataReader = createDataReader(createDescriptors(status, statement.outMetadata), attachment.encoding); + statement.dataReader = createDataReader( + createDescriptors(status, statement.outMetadata), + attachment.charsetCodec, + ); } return statement; diff --git a/packages/node-firebird-driver/src/lib/impl/attachment.ts b/packages/node-firebird-driver/src/lib/impl/attachment.ts index f5c0267..fe9b113 100644 --- a/packages/node-firebird-driver/src/lib/impl/attachment.ts +++ b/packages/node-firebird-driver/src/lib/impl/attachment.ts @@ -19,7 +19,6 @@ import { /** AbstractAttachment implementation. */ export abstract class AbstractAttachment implements Attachment { - encoding: BufferEncoding = 'utf8'; events = new Set(); statements = new Set(); transactions = new Set(); diff --git a/packages/node-firebird-driver/src/lib/impl/fb-util.ts b/packages/node-firebird-driver/src/lib/impl/fb-util.ts index 2e23c11..fa2ee21 100644 --- a/packages/node-firebird-driver/src/lib/impl/fb-util.ts +++ b/packages/node-firebird-driver/src/lib/impl/fb-util.ts @@ -132,14 +132,91 @@ export namespace charSets { export const ascii = 2; } -/** Maps a Firebird charset name to a Node.js encoding. */ -export function mapCharsetToEncoding(charset?: string): BufferEncoding { - if (!charset || charset.toUpperCase() === 'UTF8') { - return 'utf8'; - } else { - return 'latin1'; +/** Maps Firebird charset names to TextDecoder encoding labels. */ +const firebirdCharsetMap: Record = { + UTF8: 'utf-8', + NONE: 'utf-8', + ASCII: 'ascii', + WIN1250: 'windows-1250', + WIN1251: 'windows-1251', + WIN1252: 'windows-1252', + WIN1253: 'windows-1253', + WIN1254: 'windows-1254', + WIN1255: 'windows-1255', + WIN1256: 'windows-1256', + WIN1257: 'windows-1257', + WIN1258: 'windows-1258', + ISO8859_1: 'iso-8859-1', + ISO8859_2: 'iso-8859-2', + ISO8859_3: 'iso-8859-3', + ISO8859_4: 'iso-8859-4', + ISO8859_5: 'iso-8859-5', + ISO8859_6: 'iso-8859-6', + ISO8859_7: 'iso-8859-7', + ISO8859_8: 'iso-8859-8', + ISO8859_9: 'iso-8859-9', + ISO8859_13: 'iso-8859-13', + ISO8859_15: 'iso-8859-15', +}; + +/** Resolves a Firebird charset name to a TextDecoder encoding label. */ +export function resolveCharsetEncoding(charset?: string): string { + if (!charset) { + return 'utf-8'; } + const label = firebirdCharsetMap[charset.toUpperCase()]; + if (!label) { + throw new Error( + `Unsupported Firebird charset: '${charset}'. Supported: ${Object.keys(firebirdCharsetMap).join(', ')}`, + ); + } + return label; +} + +/** Codec for encoding/decoding strings in a specific charset. */ +export interface CharsetCodec { + decode(buffer: Uint8Array): string; + encode(str: string): Buffer; +} + +/** Creates a codec for encoding/decoding strings in the given charset. */ +export function createCharsetCodec(charset?: string): CharsetCodec { + const encoding = resolveCharsetEncoding(charset); + + if (encoding === 'utf-8') { + const decoder = new TextDecoder('utf-8'); + return { + decode: (buffer: Uint8Array) => decoder.decode(buffer), + encode: (str: string) => Buffer.from(str, 'utf8'), + }; + } + + const decoder = new TextDecoder(encoding); + const reverseMap = new Map(); + for (let i = 0; i < 256; i++) { + const char = decoder.decode(new Uint8Array([i])); + reverseMap.set(char, i); + } + + return { + decode: (buffer: Uint8Array) => decoder.decode(buffer), + encode: (str: string) => { + const bytes = new Uint8Array(str.length); + for (let i = 0; i < str.length; i++) { + const b = reverseMap.get(str[i]); + if (b === undefined) { + throw new Error( + `Character '${str[i]}' (U+${str[i].charCodeAt(0).toString(16).padStart(4, '0')}) ` + + `cannot be encoded in charset '${encoding}'.`, + ); + } + bytes[i] = b; + } + return Buffer.from(bytes); + }, + }; } + export function createDpb(options?: ConnectOptions | CreateDatabaseOptions): Buffer { const code = (c: number) => String.fromCharCode(c); const charSet = options?.charset ?? 'utf8'; @@ -314,7 +391,8 @@ export type DataReader = (attachment: Attachment, transaction: Transaction, buff export type ItemReader = (attachment: Attachment, transaction: Transaction, buffer: Uint8Array) => Promise; /** Creates a data reader. */ -export function createDataReader(descriptors: Descriptor[], encoding: BufferEncoding = 'utf8'): DataReader { +export function createDataReader(descriptors: Descriptor[], codec?: CharsetCodec): DataReader { + const charsetCodec = codec ?? createCharsetCodec(); const mappers = new Array(descriptors.length); for (let i = 0; i < descriptors.length; ++i) { @@ -336,9 +414,8 @@ export function createDataReader(descriptors: Descriptor[], encoding: BufferEnco case sqlTypes.SQL_VARYING: { //// TODO: none, octets const varLength = dataView.getUint16(descriptor.offset, littleEndian); - const decoder = new stringDecoder.StringDecoder(encoding); - const buf = Buffer.from(buffer.buffer, descriptor.offset + 2, varLength); - return decoder.end(buf); + const buf = new Uint8Array(buffer.buffer, descriptor.offset + 2, varLength); + return charsetCodec.decode(buf); } /*** @@ -511,7 +588,8 @@ export type ItemWriter = ( ) => Promise; /** Creates a data writer. */ -export function createDataWriter(descriptors: Descriptor[], encoding: BufferEncoding = 'utf8'): DataWriter { +export function createDataWriter(descriptors: Descriptor[], codec?: CharsetCodec): DataWriter { + const charsetCodec = codec ?? createCharsetCodec(); const mappers = new Array(descriptors.length); for (let i = 0; i < descriptors.length; ++i) { @@ -537,7 +615,7 @@ export function createDataWriter(descriptors: Descriptor[], encoding: BufferEnco case sqlTypes.SQL_VARYING: { //// TODO: none, octets const str = value as string; - const strBuffer = Buffer.from(str, encoding); + const strBuffer = charsetCodec.encode(str); const bytesArray = Uint8Array.from(strBuffer); From e98b5c24013a8d201fa3c2ef965f0d7ac27f115e Mon Sep 17 00:00:00 2001 From: Massimo Fazzolari Date: Tue, 17 Feb 2026 16:17:34 +0100 Subject: [PATCH 3/3] remove unused stringDecoder import --- packages/node-firebird-driver/src/lib/impl/fb-util.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/node-firebird-driver/src/lib/impl/fb-util.ts b/packages/node-firebird-driver/src/lib/impl/fb-util.ts index fa2ee21..d26a83f 100644 --- a/packages/node-firebird-driver/src/lib/impl/fb-util.ts +++ b/packages/node-firebird-driver/src/lib/impl/fb-util.ts @@ -1,8 +1,5 @@ import * as os from 'os'; const littleEndian = os.endianness() === 'LE'; - -import * as stringDecoder from 'string_decoder'; - import { AbstractAttachment } from './attachment'; import { decodeDate, decodeTime, encodeDate, encodeTime } from './date-time'; import { tzIdToString, tzStringToId } from './time-zones';