feat: implement safe integer mode for SQLiteCloud connection string

Author: Daniele Briggi

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@sqlitecloud/drivers",
-  "version": "1.0.779",
+  "version": "1.0.834",
   "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",

package.json

@@ -221,17 +221,17 @@ export class SQLiteCloudTlsConnection extends SQLiteCloudConnection {
               this.processCommandsData(Buffer.alloc(0))
               return
             } else {
-              const { data } = popData(decompressResults.buffer)
+              const { data } = popData(decompressResults.buffer, this.config.safe_integer_mode)
               this.processCommandsFinish?.call(this, null, data)
             }
           } else {
             if (dataType !== CMD_ROWSET_CHUNK) {
-              const { data } = popData(this.buffer)
+              const { data } = popData(this.buffer, this.config.safe_integer_mode)
               this.processCommandsFinish?.call(this, null, data)
             } else {
               const completeChunk = bufferEndsWith(this.buffer, ROWSET_CHUNKS_END)
               if (completeChunk) {
-                const parsedData = parseRowsetChunks([...this.pendingChunks, this.buffer])
+                const parsedData = parseRowsetChunks([...this.pendingChunks, this.buffer], this.config.safe_integer_mode)
                 this.processCommandsFinish?.call(this, null, parsedData)
               }
             }
@@ -241,7 +241,7 @@ export class SQLiteCloudTlsConnection extends SQLiteCloudConnection {
         // command with no explicit len so make sure that the final character is a space
         const lastChar = this.buffer.subarray(this.buffer.length - 1, this.buffer.length).toString('utf8')
         if (lastChar == ' ') {
-          const { data } = popData(this.buffer)
+          const { data } = popData(this.buffer, this.config.safe_integer_mode)
           this.processCommandsFinish?.call(this, null, data)
         }
       }

src/drivers/connection-tls.ts

@@ -3,7 +3,14 @@
 //
 
 import { SQLiteCloudRowset } from './rowset'
-import { SAFE_INTEGER_MODE, SQLiteCloudCommand, SQLiteCloudError, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types'
+import {
+  SAFE_INTEGER_MODE,
+  SQLiteCloudCommand,
+  SQLiteCloudError,
+  type SQLCloudRowsetMetadata,
+  type SQLiteCloudDataTypes,
+  type SQLiteCloudSafeIntegerMode
+} from './types'
 import { getSafeBuffer } from './safe-imports'
 
 // explicitly importing buffer library to allow cross-platform support by replacing it
@@ -125,15 +132,15 @@ export function parseError(buffer: Buffer, spaceIndex: number): never {
 }
 
 /** Parse an array of items (each of which will be parsed by type separately) */
-export function parseArray(buffer: Buffer, spaceIndex: number): SQLiteCloudDataTypes[] {
+export function parseArray(buffer: Buffer, spaceIndex: number, safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE): SQLiteCloudDataTypes[] {
   const parsedData = []
 
   const array = buffer.subarray(spaceIndex + 1, buffer.length)
   const numberOfItems = parseInt(array.subarray(0, spaceIndex - 2).toString('utf8'))
   let arrayItems = array.subarray(array.indexOf(' ') + 1, array.length)
 
   for (let i = 0; i < numberOfItems; i++) {
-    const { data, fwdBuffer: buffer } = popData(arrayItems)
+    const { data, fwdBuffer: buffer } = popData(arrayItems, safeIntegerMode)
     parsedData.push(data)
     arrayItems = buffer
   }
@@ -165,9 +172,9 @@ export function parseRowsetHeader(buffer: Buffer): { index: number; metadata: SQ
 }
 
 /** Extract column names and, optionally, more metadata out of a rowset's header */
-function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMetadata): Buffer {
+function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMetadata, safeIntegerMode: SQLiteCloudSafeIntegerMode): Buffer {
   function popForward() {
-    const { data, fwdBuffer: fwdBuffer } = popData(buffer) // buffer in parent scope
+    const { data, fwdBuffer: fwdBuffer } = popData(buffer, safeIntegerMode) // buffer in parent scope
     buffer = fwdBuffer
     return data
   }
@@ -192,16 +199,16 @@ function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMeta
 }
 
 /** Parse a regular rowset (no chunks) */
-function parseRowset(buffer: Buffer, spaceIndex: number): SQLiteCloudRowset {
+function parseRowset(buffer: Buffer, spaceIndex: number, safeIntegerMode: SQLiteCloudSafeIntegerMode): SQLiteCloudRowset {
   buffer = buffer.subarray(spaceIndex + 1, buffer.length)
 
   const { metadata, fwdBuffer } = parseRowsetHeader(buffer)
-  buffer = parseRowsetColumnsMetadata(fwdBuffer, metadata)
+  buffer = parseRowsetColumnsMetadata(fwdBuffer, metadata, safeIntegerMode)
 
   // decode each rowset item
   const data = []
   for (let j = 0; j < metadata.numberOfRows * metadata.numberOfColumns; j++) {
-    const { data: rowData, fwdBuffer } = popData(buffer)
+    const { data: rowData, fwdBuffer } = popData(buffer, safeIntegerMode)
     data.push(rowData)
     buffer = fwdBuffer
   }
@@ -223,7 +230,7 @@ export function bufferEndsWith(buffer: Buffer, suffix: string): boolean {
  * *LEN 0:VERS NROWS NCOLS DATA
  * @see https://github.com/sqlitecloud/sdk/blob/master/PROTOCOL.md#scsp-rowset-chunk
  */
-export function parseRowsetChunks(buffers: Buffer[]): SQLiteCloudRowset {
+export function parseRowsetChunks(buffers: Buffer[], safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE): SQLiteCloudRowset {
   let buffer = Buffer.concat(buffers)
   if (!bufferStartsWith(buffer, CMD_ROWSET_CHUNK) || !bufferEndsWith(buffer, ROWSET_CHUNKS_END)) {
     throw new Error('SQLiteCloudConnection.parseRowsetChunks - invalid chunks buffer')
@@ -245,14 +252,14 @@ export function parseRowsetChunks(buffers: Buffer[]): SQLiteCloudRowset {
     // first chunk? extract columns metadata
     if (chunkIndex === 1) {
       metadata = chunkMetadata
-      buffer = parseRowsetColumnsMetadata(buffer, metadata)
+      buffer = parseRowsetColumnsMetadata(buffer, metadata, safeIntegerMode)
     } else {
       metadata.numberOfRows += chunkMetadata.numberOfRows
     }
 
     // extract single rowset row
     for (let k = 0; k < chunkMetadata.numberOfRows * metadata.numberOfColumns; k++) {
-      const { data: itemData, fwdBuffer } = popData(buffer)
+      const { data: itemData, fwdBuffer } = popData(buffer, safeIntegerMode)
       data.push(itemData)
       buffer = fwdBuffer
     }
@@ -276,7 +283,10 @@ function popIntegers(buffer: Buffer, numberOfIntegers = 1): { data: number[]; fw
 }
 
 /** Parse command, extract its data, return the data and the buffer moved to the first byte after the command */
-export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCloudRowset; fwdBuffer: Buffer } {
+export function popData(
+  buffer: Buffer,
+  safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE
+): { data: SQLiteCloudDataTypes | SQLiteCloudRowset; fwdBuffer: Buffer } {
   function popResults(data: any) {
     const fwdBuffer = buffer.subarray(commandEnd)
     return { data, fwdBuffer }
@@ -307,10 +317,10 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl
     case CMD_INT:
       // SQLite uses 64-bit INTEGER, but JS uses 53-bit Number
       const value = BigInt(buffer.subarray(1, spaceIndex).toString())
-      if (SAFE_INTEGER_MODE === 'bigint') {
+      if (safeIntegerMode === 'bigint') {
         return popResults(value)
       }
-      if (SAFE_INTEGER_MODE === 'mixed') {
+      if (safeIntegerMode === 'mixed') {
         if (value <= BigInt(Number.MIN_SAFE_INTEGER) || BigInt(Number.MAX_SAFE_INTEGER) <= value) {
           return popResults(value)
         }
@@ -333,9 +343,9 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl
     case CMD_BLOB:
       return popResults(buffer.subarray(spaceIndex + 1, commandEnd))
     case CMD_ARRAY:
-      return popResults(parseArray(buffer, spaceIndex))
+      return popResults(parseArray(buffer, spaceIndex, safeIntegerMode))
     case CMD_ROWSET:
-      return popResults(parseRowset(buffer, spaceIndex))
+      return popResults(parseRowset(buffer, spaceIndex, safeIntegerMode))
     case CMD_ERROR:
       parseError(buffer, spaceIndex) // throws custom error
       break

src/drivers/protocol.ts

@@ -19,9 +19,14 @@ export const DEFAULT_PORT = 8860
  *  (inlcuding `lastID` from WRITE statements)
  * mixed - use BigInt and Number types depending on the value size
  */
-export let SAFE_INTEGER_MODE = 'number'
+export type SQLiteCloudSafeIntegerMode = 'number' | 'bigint' | 'mixed'
+
+export let SAFE_INTEGER_MODE: SQLiteCloudSafeIntegerMode = 'number'
 if (typeof process !== 'undefined') {
-   SAFE_INTEGER_MODE = process.env['SAFE_INTEGER_MODE']?.toLowerCase() || 'number'
+  const mode = process.env['SAFE_INTEGER_MODE']?.toLowerCase()
+  if (mode === 'bigint' || mode === 'mixed' || mode === 'number') {
+    SAFE_INTEGER_MODE = mode
+  }
 }
 if (SAFE_INTEGER_MODE == 'bigint') {
   console.debug('BigInt mode: Using Number for all INTEGER values from SQLite, including meta information from WRITE statements.')
@@ -79,6 +84,8 @@ export interface SQLiteCloudConfig {
   maxrows?: number
   /** Server should limit total number of rows in a set to maxRowset */
   maxrowset?: number
+  /** How SQLite 64-bit INTEGER values are returned: number, bigint or mixed. Defaults to SAFE_INTEGER_MODE env var, then number */
+  safe_integer_mode?: SQLiteCloudSafeIntegerMode
 
   /** Custom options and configurations for tls socket, eg: additional certificates */
   tlsoptions?: tls.ConnectionOptions

src/drivers/types.ts

@@ -2,7 +2,16 @@
 // utilities.ts - utility methods to manipulate SQL statements
 //
 
-import { DEFAULT_PORT, DEFAULT_TIMEOUT, SQLiteCloudArrayType, SQLiteCloudConfig, SQLiteCloudDataTypes, SQLiteCloudError } from './types'
+import {
+  DEFAULT_PORT,
+  DEFAULT_TIMEOUT,
+  SAFE_INTEGER_MODE,
+  SQLiteCloudArrayType,
+  SQLiteCloudConfig,
+  SQLiteCloudDataTypes,
+  SQLiteCloudError,
+  SQLiteCloudSafeIntegerMode
+} from './types'
 import { getSafeURL } from './safe-imports'
 
 // explicitly importing these libraries to allow cross-platform support by replacing them
@@ -174,6 +183,7 @@ export function validateConfiguration(config: SQLiteCloudConfig): SQLiteCloudCon
   config.verbose = parseBoolean(config.verbose)
   config.noblob = parseBoolean(config.noblob)
   config.compression = config.compression != undefined && config.compression != null ? parseBoolean(config.compression) : true // default: true
+  config.safe_integer_mode = parseSafeIntegerMode(config.safe_integer_mode || SAFE_INTEGER_MODE)
 
   config.create = parseBoolean(config.create)
   config.non_linearizable = parseBoolean(config.non_linearizable)
@@ -242,6 +252,7 @@ export function parseconnectionstring(connectionstring: string): SQLiteCloudConf
       maxdata: options.maxdata ? parseInt(options.maxdata) : undefined,
       maxrows: options.maxrows ? parseInt(options.maxrows) : undefined,
       maxrowset: options.maxrowset ? parseInt(options.maxrowset) : undefined,
+      safe_integer_mode: options.safe_integer_mode ? parseSafeIntegerMode(options.safe_integer_mode) : undefined,
       usewebsocket: options.usewebsocket ? parseBoolean(options.usewebsocket) : undefined,
       verbose: options.verbose ? parseBoolean(options.verbose) : undefined
     }
@@ -278,3 +289,12 @@ export function parseBooleanToZeroOne(value: string | boolean | null | undefined
   }
   return value ? 1 : 0
 }
+
+/** Parse 64-bit integer handling mode */
+export function parseSafeIntegerMode(value: string | SQLiteCloudSafeIntegerMode | null | undefined): SQLiteCloudSafeIntegerMode {
+  const mode = value?.toLowerCase()
+  if (mode === 'number' || mode === 'bigint' || mode === 'mixed') {
+    return mode
+  }
+  return 'number'
+}

src/drivers/utilities.ts

@@ -11,6 +11,7 @@ export { Database } from './drivers/database'
 export { SQLiteCloudConnection } from './drivers/connection'
 export {
   type SQLiteCloudConfig,
+  type SQLiteCloudSafeIntegerMode,
   type SQLCloudRowsetMetadata,
   SQLiteCloudError,
   type ResultsCallback,

src/index.ts

@@ -2,7 +2,7 @@
 // protocol.test.ts
 //
 
-import { formatCommand, parseRowsetChunks } from '../src/drivers/protocol'
+import { formatCommand, parseRowsetChunks, popData } from '../src/drivers/protocol'
 import { SQLiteCloudCommand } from '../src/drivers/types'
 
 // response sent by the server when we TEST ROWSET_CHUNK
@@ -31,6 +31,30 @@ describe('parseRowsetChunks', () => {
   })
 })
 
+describe('Safe integer mode', () => {
+  it('should return numbers by default', () => {
+    const { data } = popData(Buffer.from(':9007199254740992 '))
+    expect(data).toBe(9007199254740992)
+    expect(typeof data).toBe('number')
+  })
+
+  it('should return bigint when mode is bigint', () => {
+    const { data } = popData(Buffer.from(':42 '), 'bigint')
+    expect(data).toBe(BigInt(42))
+    expect(typeof data).toBe('bigint')
+  })
+
+  it('should return bigint only for unsafe integers when mode is mixed', () => {
+    const small = popData(Buffer.from(':42 '), 'mixed')
+    const large = popData(Buffer.from(':9007199254740992 '), 'mixed')
+
+    expect(small.data).toBe(42)
+    expect(typeof small.data).toBe('number')
+    expect(large.data).toBe(BigInt('9007199254740992'))
+    expect(typeof large.data).toBe('bigint')
+  })
+})
+
 const testCases = [
   { query: "SELECT 'hello world'", parameters: [], expected: "+20 SELECT 'hello world'" },
   {

test/protocol.test.ts

@@ -3,7 +3,7 @@
 //
 
 import { SQLiteCloudError } from '../src/index'
-import { getInitializationCommands, parseconnectionstring, sanitizeSQLiteIdentifier } from '../src/drivers/utilities'
+import { getInitializationCommands, parseconnectionstring, sanitizeSQLiteIdentifier, validateConfiguration } from '../src/drivers/utilities'
 import { getTestingDatabaseName } from './shared'
 
 import { expect, describe, it } from '@jest/globals'
@@ -163,6 +163,13 @@ describe('parseconnectionstring', () => {
     expect(config.timeout).toBe(123)
   })
 
+  it('should parse connection with safe integer mode', () => {
+    const connectionstring = `sqlitecloud://host:1234/database?apikey=xxx&safe_integer_mode=bigint`
+    const config = parseconnectionstring(connectionstring)
+
+    expect(config.safe_integer_mode).toBe('bigint')
+  })
+
   it('expect error when both user/pass and api key are set', () => {
     const connectionstring = 'sqlitecloud://user:password@host:1234/database?apikey=yyy'
     expect(() => parseconnectionstring(connectionstring)).toThrowError('Choose between apikey, token or username/password')
@@ -179,6 +186,19 @@ describe('parseconnectionstring', () => {
   })
 })
 
+describe('validateConfiguration()', () => {
+  it('should use safe integer mode from config', () => {
+    const config = validateConfiguration({
+      username: 'user',
+      password: 'password',
+      host: 'host',
+      safe_integer_mode: 'mixed'
+    })
+
+    expect(config.safe_integer_mode).toBe('mixed')
+  })
+})
+
 describe('getTestingDatabaseName', () => {
   it('should generate readable database names', () => {
     const database = getTestingDatabaseName('benchkmark')
@@ -218,4 +238,4 @@ describe('getInitializationCommands()', () => {
     expect(result).toContain('AUTH TOKEN mytoken;')
     expect(result).not.toContain('AUTH APIKEY')
   })
-})
+})