From 3362f6bf77cf6687993a58d64721e1e1f2f7a755 Mon Sep 17 00:00:00 2001 From: Alan McCann Date: Thu, 13 Aug 2026 06:25:54 -0400 Subject: [PATCH] Add EXQLITE_EXTRA_SRC: statically compile SQLite extensions into the NIF Some targets cannot use run-time loadable extensions at all: iOS forbids dynamic code loading, and hardened deployments often disable load_extension. Shipping per-architecture .so files is also a maintenance burden that static compilation removes. EXQLITE_EXTRA_SRC takes a space-separated list of C files compiled and linked into the NIF alongside the bundled sqlite3.c. Paired with the existing EXQLITE_SYSTEM_CFLAGS (-DSQLITE_CORE=1 -DSQLITE_EXTRA_INIT=...), an extension such as sqlite-vec registers itself on every connection with no load_extension call anywhere. - Only applies when compiling the bundled SQLite; ignored under EXQLITE_USE_SYSTEM (linking is the system library's business there). - No-op when the variable is unset: OBJ and the pattern rules are unchanged, verified by building and running with and without the hook. - Extra objects are namespaced as extra_.o in BUILD, with sources resolved via vpath, so out-of-tree paths (deps, vendored dirs) work. - POSIX make only for now; Makefile.win untouched. Verified end-to-end against this branch via a path dep and make_env only (no fork consumer patches): sqlite-vec v0.1.9 statically linked into the bundled 3.53.4, SELECT vec_distance_cosine(vec_f32('[1,0]'), vec_f32('[0,1]')) returning 1.0 and the 45-degree case returning 1 - 1/sqrt(2). Also verified on iOS (device-SDK static archive + execution in the simulator) in the migration that motivated this, where load_extension was not an option. --- CHANGELOG.md | 1 + Makefile | 18 ++++++++++++++++++ README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c90ad74..da37e1bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - changed: Update sqlite to `3.53.4`. +- added: `EXQLITE_EXTRA_SRC` build variable to statically compile SQLite extensions (e.g. sqlite-vec) into the NIF, for targets where `load_extension` is unavailable (iOS) or undesirable. ## v0.39.0 diff --git a/Makefile b/Makefile index c750b918..3349d569 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,20 @@ ARCHIVE_NAME = $(PREFIX)/sqlite3_nif.a OBJ = $(SRC:c_src/%.c=$(BUILD)/%.o) +# Statically compile additional C sources (for example SQLite extensions such +# as sqlite-vec or spellfix1) into the NIF. Space-separated list of file paths. +# Pair with EXQLITE_SYSTEM_CFLAGS to define SQLITE_EXTRA_INIT so the extension +# registers itself on every connection - see "Statically compiling SQLite3 +# extensions" in the README. Only meaningful when compiling the bundled SQLite +# (ignored under EXQLITE_USE_SYSTEM, where linking is the system library's job). +ifneq ($(EXQLITE_EXTRA_SRC),) +ifeq ($(EXQLITE_USE_SYSTEM),) + EXTRA_OBJ = $(addprefix $(BUILD)/extra_,$(notdir $(EXQLITE_EXTRA_SRC:.c=.o))) + OBJ += $(EXTRA_OBJ) + vpath %.c $(sort $(dir $(EXQLITE_EXTRA_SRC))) +endif +endif + ifneq ($(CROSSCOMPILE),) ifeq ($(CROSSCOMPILE), Android) CFLAGS:=$(filter-out -O2,$(CFLAGS)) @@ -144,6 +158,10 @@ $(BUILD)/%.o: c_src/%.c @echo " CC $(notdir $@)" $(CC) -c $(ERL_CFLAGS) $(CFLAGS) -MMD -MP -o $@ $< +$(BUILD)/extra_%.o: %.c + @echo " CC $(notdir $@)" + $(CC) -c $(ERL_CFLAGS) $(CFLAGS) -MMD -MP -o $@ $< + # Include dependency files for automatic header tracking -include $(OBJ:.o=.d) diff --git a/README.md b/README.md index 2b9d433a..72ba7118 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,55 @@ See [Exqlite.Connection.connect/1](https://hexdocs.pm/exqlite/Exqlite.Connection for more information. When using extensions for SQLite3, they must be compiled for the environment you are targeting. +### Statically compiling SQLite3 extensions + +Run-time loading covers most cases, but some targets forbid `load_extension` +entirely (iOS refuses dynamic code loading; some hardened Linux deployments +disable it), and shipping one `.so` per architecture is its own maintenance +burden. For those cases the build supports compiling extension sources +directly into the NIF: + +```elixir +config :exqlite, + force_build: true, + make_env: %{ + "EXQLITE_EXTRA_SRC" => "/path/to/sqlite-vec.c /path/to/vec_init_shim.c", + "EXQLITE_SYSTEM_CFLAGS" => "-DSQLITE_CORE=1 -DSQLITE_EXTRA_INIT=exqlite_vec_init" + } +``` + +where the shim registers the extension for every future connection via +[`SQLITE_EXTRA_INIT`](https://sqlite.org/compile.html#extra_init) and +`sqlite3_auto_extension`: + +```c +/* vec_init_shim.c */ +#include "sqlite3.h" +#include "sqlite-vec.h" + +int exqlite_vec_init(const char *unused) { + (void)unused; + return sqlite3_auto_extension((void (*)(void))sqlite3_vec_init); +} +``` + +```elixir +{:ok, conn} = Exqlite.Sqlite3.open(":memory:") +{:ok, stmt} = Exqlite.Sqlite3.prepare(conn, "SELECT vec_distance_cosine(vec_f32('[1,0]'), vec_f32('[0,1]'))") +{:row, [1.0]} = Exqlite.Sqlite3.step(conn, stmt) +``` + +Notes: + +- `-DSQLITE_CORE=1` makes the extension compile against the core API instead + of the loadable-extension shim. +- Only applies when compiling the bundled SQLite (it is ignored under + `EXQLITE_USE_SYSTEM`), and requires `force_build: true` so a precompiled + NIF is not fetched instead. +- Extra sources are looked up by basename, so two files with the same + basename in different directories will collide. +- POSIX make only for now (`Makefile.win` does not implement it). + ## Why SQLite3 I needed an Ecto3 adapter to store time series data for a personal project. I