fix: lazily load database clients via dynamic import()

src/drivers/mysql/attribute.ts

@@ -1,8 +1,10 @@
-import { escape, escapeId } from 'sqlstring';
+import SqlString from 'sqlstring';
 
 import Attribute from '../abstract/attribute';
 import DataTypes from './data_types';
 
+const { escape, escapeId } = SqlString;
+
 class MysqlAttribute extends Attribute {
   constructor(name: string, params?: any, opts?: any) {
     super(name, params, opts);

src/drivers/mysql/index.ts

@@ -24,39 +24,35 @@ type SchemaInfo = Record<string, SchemaColumn[]>;
 
 type MysqlPool = any;
 type MysqlConnection = any;
+type MysqlOptions = ConnectOptions & {
+  client?: string;
+  appName?: string;
+  connectionLimit?: number;
+  connectTimeout?: number;
+  charset?: string;
+  stringifyObjects?: boolean;
+  decimalNumbers?: boolean;
+  supportBigNumbers?: boolean;
+  bigNumberStrings?: boolean;
+};
 
 class MysqlDriver extends AbstractDriver {
   static Spellbook = Spellbook;
   static Attribute = Attribute;
   static DataTypes = DataTypes;
 
   declare pool: MysqlPool;
+  private poolPromise?: Promise<MysqlPool>;
 
-  constructor(opts: ConnectOptions & { client?: string; appName?: string }) {
+  constructor(opts: MysqlOptions) {
     super(opts);
     this.type = 'mysql';
-    this.pool = this.createPool(opts);
     this.Attribute = (this.constructor as typeof MysqlDriver).Attribute;
     this.DataTypes = (this.constructor as typeof MysqlDriver).DataTypes;
     this.spellbook = new (this.constructor as typeof MysqlDriver).Spellbook();
-
-    this.escape = this.pool.escape.bind(this.pool);
-    this.escapeId = this.pool.escapeId;
   }
 
-  createPool(
-    opts: ConnectOptions & {
-      client?: string;
-      appName?: string;
-      connectionLimit?: number;
-      connectTimeout?: number;
-      charset?: string;
-      stringifyObjects?: boolean;
-      decimalNumbers?: boolean;
-      supportBigNumbers?: boolean;
-      bigNumberStrings?: boolean;
-    },
-  ): MysqlPool {
+  async createPool(opts: MysqlOptions): Promise<MysqlPool> {
     const database = opts.appName || opts.database;
     const client = opts.client || 'mysql';
     const {
@@ -77,7 +73,9 @@ class MysqlDriver extends AbstractDriver {
       console.warn(`[leoric] mysql client "${client}" not tested`);
     }
 
-    return require(client).createPool({
+    const module = await import(client);
+    const mysql = module.default ?? module;
+    return mysql.createPool({
       connectionLimit,
       connectTimeout,
       host,
@@ -93,9 +91,26 @@ class MysqlDriver extends AbstractDriver {
     });
   }
 
-  getConnection(): Promise<MysqlConnection> {
+  private async getPool(): Promise<MysqlPool> {
+    if (!this.poolPromise) {
+      this.poolPromise = this.createPool(this.options as MysqlOptions).then((pool) => {
+        this.pool = pool;
+        return pool;
+      });
+    }
+
+    try {
+      return await this.poolPromise;
+    } catch (error) {
+      this.poolPromise = undefined;
+      throw error;
+    }
+  }
+
+  async getConnection(): Promise<MysqlConnection> {
+    const pool = await this.getPool();
     return new Promise((resolve, reject) => {
-      this.pool.getConnection((err: Error | null, connection: MysqlConnection) => {
+      pool.getConnection((err: Error | null, connection: MysqlConnection) => {
         if (err) {
           reject(err);
         } else {

src/drivers/postgres/index.ts

@@ -19,10 +19,11 @@ class PostgresDriver extends AbstractDriver {
   static Attribute = Attribute;
   static DataTypes = DataTypes;
 
+  private poolPromise?: Promise<any>;
+
   constructor(opts: ConnectOptions) {
     super(opts);
     this.type = 'postgres';
-    this.pool = this.createPool(opts);
     this.Attribute = (this.constructor as typeof PostgresDriver).Attribute;
     this.DataTypes = (this.constructor as typeof PostgresDriver).DataTypes;
     this.spellbook = new (this.constructor as typeof PostgresDriver).Spellbook();
@@ -31,16 +32,34 @@ class PostgresDriver extends AbstractDriver {
     this.escapeId = escapeId;
   }
 
-  createPool(opts: any) {
+  async createPool(opts: ConnectOptions) {
     const { host, port, user, password, database } = opts;
-    require('./type_parser');
-    const { Pool } = require('pg');
-    return new Pool({ host, port, user, password, database });
+    const client = opts.client || 'pg';
+    if (client === 'pg') await import('./type_parser');
+    const module = await import(client);
+    const pg = module.default ?? module;
+    return new pg.Pool({ host, port, user, password, database });
+  }
+
+  private async getPool() {
+    if (!this.poolPromise) {
+      this.poolPromise = this.createPool(this.options).then((pool) => {
+        this.pool = pool;
+        return pool;
+      });
+    }
+
+    try {
+      return await this.poolPromise;
+    } catch (error) {
+      this.poolPromise = undefined;
+      throw error;
+    }
   }
 
   async getConnection() {
     // pg Pool supports connect() with callback or promise; cast to any for TS
-    return await (this.pool as any).connect();
+    return await (await this.getPool()).connect();
   }
 
   async query(query: any, values?: any, spell: any = {}) {
@@ -118,7 +137,7 @@ class PostgresDriver extends AbstractDriver {
     ORDER BY columns.ordinal_position ASC
     `);
 
-    const { pool } = this as any;
+    const pool = await this.getPool();
     const { rows } = await pool.query(text, [database, tableList]);
     const schemaInfo: any = {};
 

src/drivers/sqlite/pool.ts

@@ -15,7 +15,8 @@ export interface PoolConnection extends Connection {
 
 class Pool extends EventEmitter {
   options: PoolOptions;
-  client: any;
+  client?: any;
+  private clientPromise?: Promise<any>;
   connections: PoolConnection[];
   queue: Array<() => void>;
   connectionLimit: number;
@@ -29,21 +30,35 @@ class Pool extends EventEmitter {
       client: (opts as any).client || 'sqlite3',
     };
 
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const client = require(options.client!);
-    // Turn on stack trace capturing otherwise the output is useless
-    // - https://github.com/mapbox/node-sqlite3/wiki/Debugging
-    if (options.trace) client.verbose();
-
     this.options = options;
-    this.client = client;
     this.connections = [];
     this.queue = [];
     this.connectionLimit = options.connectionLimit || 10;
   }
 
+  private async loadClient(): Promise<any> {
+    if (!this.clientPromise) {
+      this.clientPromise = import(this.options.client ?? 'sqlite3').then((module) => {
+        const client = module.default ?? module;
+        // Turn on stack trace capturing otherwise the output is useless
+        // - https://github.com/mapbox/node-sqlite3/wiki/Debugging
+        if (this.options.trace) client.verbose();
+        this.client = client;
+        return client;
+      });
+    }
+
+    try {
+      return await this.clientPromise;
+    } catch (error) {
+      this.clientPromise = undefined;
+      throw error;
+    }
+  }
+
   async getConnection(): Promise<PoolConnection> {
-    const { connections, queue, client, connectionLimit } = this;
+    const client = await this.loadClient();
+    const { connections, queue, connectionLimit } = this;
     for (const connection of connections) {
       if (connection.idle) {
         connection.idle = false;

test/fixtures/dynamic-client.js

@@ -0,0 +1,67 @@
+'use strict';
+
+let verboseCalls = 0;
+let createPoolCalls = 0;
+let postgresPoolCalls = 0;
+
+class Database {
+  configure() {}
+  close(callback) {
+    callback(null);
+  }
+}
+
+class Pool {
+  constructor(options) {
+    postgresPoolCalls += 1;
+    this.options = options;
+  }
+
+  async connect() {
+    return { release() {} };
+  }
+
+  async query() {
+    return { rows: [] };
+  }
+}
+
+function createPool(options) {
+  createPoolCalls += 1;
+  return {
+    options,
+    getConnection(callback) {
+      callback(null, { release() {} });
+    },
+  };
+}
+
+function verbose() {
+  verboseCalls += 1;
+  return module.exports;
+}
+
+function reset() {
+  verboseCalls = 0;
+  createPoolCalls = 0;
+  postgresPoolCalls = 0;
+}
+
+module.exports = {
+  Database,
+  OPEN_READWRITE: 1,
+  OPEN_CREATE: 2,
+  Pool,
+  createPool,
+  verbose,
+  reset,
+  get verboseCalls() {
+    return verboseCalls;
+  },
+  get createPoolCalls() {
+    return createPoolCalls;
+  },
+  get postgresPoolCalls() {
+    return postgresPoolCalls;
+  },
+};

test/unit/drivers/dynamic-client.test.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const assert = require('assert').strict;
+const path = require('path');
+const { MysqlDriver, PostgresDriver } = require('../../../src/drivers');
+const SqlitePool = require('../../../src/drivers/sqlite/pool').default;
+
+const clientPath = path.resolve(__dirname, '../../fixtures/dynamic-client.js');
+const client = require(clientPath);
+
+describe('=> dynamically loaded database clients', function() {
+  beforeEach(function() {
+    client.reset();
+  });
+
+  it('should lazily load a configured SQLite client once', async function() {
+    const pool = new SqlitePool({ client: clientPath, database: ':memory:' });
+    assert.equal(pool.client, undefined);
+
+    await Promise.all([ pool.getConnection(), pool.getConnection() ]);
+
+    assert.equal(pool.client, client);
+    assert.equal(client.verboseCalls, 1);
+    await pool.end();
+  });
+
+  it('should apply SQLite connection limit while client is loading', async function() {
+    const pool = new SqlitePool({ client: clientPath, connectionLimit: 1, database: ':memory:' });
+    const firstPromise = pool.getConnection();
+    const secondPromise = pool.getConnection();
+    const first = await firstPromise;
+    first.release();
+    const second = await secondPromise;
+
+    assert.equal(pool.connections.length, 1);
+    assert.equal(second, first);
+    await pool.end();
+  });
+
+  it('should lazily create a configured MySQL pool once', async function() {
+    const driver = new MysqlDriver({ client: clientPath });
+    assert.deepEqual(driver.pool, {});
+
+    await Promise.all([ driver.getConnection(), driver.getConnection() ]);
+
+    assert.equal(client.createPoolCalls, 1);
+    assert.equal(driver.pool.options.connectionLimit, undefined);
+    assert.equal(driver.escape("O'Reilly"), "'O\\'Reilly'");
+  });
+
+  it('should lazily create a configured PostgreSQL pool once', async function() {
+    const driver = new PostgresDriver({ client: clientPath });
+    assert.deepEqual(driver.pool, {});
+
+    await Promise.all([ driver.getConnection(), driver.getConnection() ]);
+
+    assert.equal(client.postgresPoolCalls, 1);
+  });
+});

test/unit/realm.test.js

@@ -77,9 +77,10 @@ describe('=> Realm', () => {
     });
 
     it('should be able to switch client with opts.dialectModule', async () => {
-      assert.throws(function() {
-        new Realm({ dialectModulePath: 'maria' });
-      }, /Cannot find module 'maria'/);
+      await assert.rejects(
+        async () => await new Realm({ dialectModulePath: 'maria' }).driver.getConnection(),
+        /Cannot find (module|package) 'maria'/,
+      );
 
       const realm = new Realm({
         dialectModulePath: 'mysql2',
@@ -89,6 +90,8 @@ describe('=> Realm', () => {
         database: 'leoric',
         models: path.resolve(__dirname, '../models')
       });
+      const connection = await realm.driver.getConnection();
+      connection.release();
       assert.equal(realm.driver.pool.constructor.name, 'Pool');
     });