diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4840d74..1151ed6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,11 +14,7 @@ Fixes SC-XXXXX ### Acceptance criteria -This PR will be merged without review. - - ### Author checklist @@ -27,11 +23,4 @@ This PR will be merged without review. - [ ] I have added new test fixtures as needed to support added tests - [ ] I have added or updated the documentation - [ ] I have run go generate to update generated code - \ No newline at end of file diff --git a/README.md b/README.md index c742b63..2eb656b 100644 --- a/README.md +++ b/README.md @@ -12,18 +12,18 @@ Import [`go.rtnl.ai/tidal`](https://pkg.go.dev/go.rtnl.ai/tidal) in application ## Packages -| Import | Purpose | -| --- | --- | -| [`go.rtnl.ai/tidal`](https://pkg.go.dev/go.rtnl.ai/tidal) | Main entry point — re-exports connections, models, filters, and CRUD | -| [`go.rtnl.ai/tidal/conn`](https://pkg.go.dev/go.rtnl.ai/tidal/conn) | `DB`, `Tx`, `Open`, `Wrap`, [`Beginner`](https://pkg.go.dev/go.rtnl.ai/tidal/conn#Beginner) | -| [`go.rtnl.ai/tidal/model`](https://pkg.go.dev/go.rtnl.ai/tidal/model) | `Model`, `BaseModel`, `Operation` | -| [`go.rtnl.ai/tidal/filter`](https://pkg.go.dev/go.rtnl.ai/tidal/filter) | `Filter`, `CustomFilter`, list-query pagination | -| [`go.rtnl.ai/tidal/store`](https://pkg.go.dev/go.rtnl.ai/tidal/store) | `CRUD`, `Cursor`, query generation | -| [`go.rtnl.ai/tidal/bind`](https://pkg.go.dev/go.rtnl.ai/tidal/bind) | `:name` placeholder rewriting | -| [`go.rtnl.ai/tidal/migrations`](https://pkg.go.dev/go.rtnl.ai/tidal/migrations) | Versioned SQL schema migrations | -| [`go.rtnl.ai/tidal/fields`](https://pkg.go.dev/go.rtnl.ai/tidal/fields) | JSON, string-array, and normalized timestamp column types for `Model` structs | -| [`go.rtnl.ai/tidal/suite`](https://pkg.go.dev/go.rtnl.ai/tidal/suite) | Database test harness, `ConformsCRUD`, shared `testdata`, and integration tests | -| [`go.rtnl.ai/tidal/suite/fixtures`](https://pkg.go.dev/go.rtnl.ai/tidal/suite/fixtures) | SQL fixture loader for test suites (`fixtures.File`) | +| Import | Purpose | +| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| [`go.rtnl.ai/tidal`](https://pkg.go.dev/go.rtnl.ai/tidal) | Main entry point — re-exports connections, models, filters, and CRUD | +| [`go.rtnl.ai/tidal/conn`](https://pkg.go.dev/go.rtnl.ai/tidal/conn) | `DB`, `Tx`, `Open`, `Wrap`, [`Beginner`](https://pkg.go.dev/go.rtnl.ai/tidal/conn#Beginner) | +| [`go.rtnl.ai/tidal/model`](https://pkg.go.dev/go.rtnl.ai/tidal/model) | `Model`, `BaseModel`, `Operation` | +| [`go.rtnl.ai/tidal/filter`](https://pkg.go.dev/go.rtnl.ai/tidal/filter) | `Filter`, `CustomFilter`, list-query pagination | +| [`go.rtnl.ai/tidal/store`](https://pkg.go.dev/go.rtnl.ai/tidal/store) | `CRUD`, `Cursor`, query generation | +| [`go.rtnl.ai/tidal/bind`](https://pkg.go.dev/go.rtnl.ai/tidal/bind) | `:name` placeholder rewriting | +| [`go.rtnl.ai/tidal/migrations`](https://pkg.go.dev/go.rtnl.ai/tidal/migrations) | Versioned SQL schema migrations | +| [`go.rtnl.ai/tidal/fields`](https://pkg.go.dev/go.rtnl.ai/tidal/fields) | JSON, string-array, and normalized timestamp column types for `Model` structs | +| [`go.rtnl.ai/tidal/suite`](https://pkg.go.dev/go.rtnl.ai/tidal/suite) | Database test harness, `ConformsCRUD`, shared `testdata`, and integration tests | +| [`go.rtnl.ai/tidal/suite/fixtures`](https://pkg.go.dev/go.rtnl.ai/tidal/suite/fixtures) | SQL fixture loader for test suites (`fixtures.File`) | ## Connecting @@ -95,19 +95,19 @@ Connection URLs are parsed by [`go.rtnl.ai/x/dsn`](https://pkg.go.dev/go.rtnl.ai ### Shared -| Option | Description | -| --- | --- | +| Option | Description | +| ---------- | ---------------------------------------------------------------------------------------------------------------------------------- | | `readonly` | When `true`, [`DB.BeginTx`](https://pkg.go.dev/go.rtnl.ai/tidal#DB.BeginTx) defaults to read-only transactions and rejects writes. | ### Postgres Tidal opens Postgres through [pgx](https://github.com/jackc/pgx) (`database/sql` stdlib driver). -| Option | Description | -| --- | --- | -| `max_idle_conns` | `database/sql` pool setting (default `8`). Removed from the URL before connecting. | -| `max_open_conns` | `database/sql` pool setting (default `16`). Removed from the URL before connecting. | -| `conn_max_lifetime` | `database/sql` pool setting (default `1h`). Removed from the URL before connecting. | +| Option | Description | +| -------------------- | ------------------------------------------------------------------------------------ | +| `max_idle_conns` | `database/sql` pool setting (default `8`). Removed from the URL before connecting. | +| `max_open_conns` | `database/sql` pool setting (default `16`). Removed from the URL before connecting. | +| `conn_max_lifetime` | `database/sql` pool setting (default `1h`). Removed from the URL before connecting. | | `conn_max_idle_time` | `database/sql` pool setting (default `30m`). Removed from the URL before connecting. | All other query parameters are forwarded to Postgres as normal connection options — see [dsn](https://pkg.go.dev/go.rtnl.ai/x/dsn). @@ -116,8 +116,8 @@ On connect, tidal registers a pgx `timestamptz` codec so values scanned into `ti ### SQLite3 -| Option | Description | -| --- | --- | +| Option | Description | +| ---------- | -------------------------------------------------------------------------- | | `readonly` | Same as above. SQLite read-only mode is enforced at the transaction level. | The database file path comes from the DSN path (`sqlite3:///path/to/db.sqlite`). Tidal does not set SQLite pragmas on `Open`; run `PRAGMA foreign_keys = on` or `PRAGMA query_only = on` after connect in application code when you need them. @@ -146,23 +146,44 @@ cursor, err := crud.List(tx, tidal.OrderBy("name").Limit(10)) users, err := cursor.List() ``` -[`Filter`](https://pkg.go.dev/go.rtnl.ai/tidal#Filter) builds ANSI SQL `WHERE`, `ORDER BY`, `LIMIT`, and `OFFSET` clauses. Calling `Where` replaces any previous WHERE clause (like `OrderBy`, `Limit`, and `Offset`). `And`, `Or`, `AndGroup`, and `OrGroup` append to the current WHERE clause. +[`Filter`](https://pkg.go.dev/go.rtnl.ai/tidal#Filter) builds ANSI SQL `WHERE`, `ORDER BY`, `LIMIT`, and `OFFSET` clauses. Calling `Where` appends an `AND` condition to the current WHERE clause, like `And`. Use `ReplaceWhere` to intentionally discard the current WHERE expression; `ClearWhere` removes the WHERE expression and its parameters while retaining the other clauses. `Clear` clears the filter completely. `Clone` copies a filter before further mutation. Start a filter with a constructor instead of the `&tidal.Filter{}` literal. Each constructor returns a `*Filter` you can keep chaining from: -| Constructor | Creates a filter with | -| --- | --- | -| [`tidal.NewFilter()`](https://pkg.go.dev/go.rtnl.ai/tidal#NewFilter) | no clauses (empty filter) | +| Constructor | Creates a filter with | +| ---------------------------------------------------------------------------- | ---------------------------- | +| [`tidal.NewFilter()`](https://pkg.go.dev/go.rtnl.ai/tidal#NewFilter) | no clauses (empty filter) | | [`tidal.Where(field, op, value)`](https://pkg.go.dev/go.rtnl.ai/tidal#Where) | an initial `WHERE` condition | -| [`tidal.OrderBy(columns...)`](https://pkg.go.dev/go.rtnl.ai/tidal#OrderBy) | an `ORDER BY` clause | -| [`tidal.Limit(n)`](https://pkg.go.dev/go.rtnl.ai/tidal#Limit) | a `LIMIT` clause | -| [`tidal.Offset(n)`](https://pkg.go.dev/go.rtnl.ai/tidal#Offset) | an `OFFSET` clause | +| [`tidal.OrderBy(columns...)`](https://pkg.go.dev/go.rtnl.ai/tidal#OrderBy) | an `ORDER BY` clause | +| [`tidal.Limit(n)`](https://pkg.go.dev/go.rtnl.ai/tidal#Limit) | a `LIMIT` clause | +| [`tidal.Offset(n)`](https://pkg.go.dev/go.rtnl.ai/tidal#Offset) | an `OFFSET` clause | -These re-export [`filter.New`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#New), [`filter.Where`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Where), [`filter.OrderBy`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#OrderBy), [`filter.Limit`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Limit), and [`filter.Offset`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Offset) (root uses `NewFilter` because `tidal.New` builds a CRUD store). Import `go.rtnl.ai/tidal/filter` to call them under the `filter.` prefix instead. +These re-export [`filter.New`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#New), [`filter.Where`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Where), [`filter.Subselect`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Subselect), [`filter.OrderBy`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#OrderBy), [`filter.Limit`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Limit), and [`filter.Offset`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Offset) (root uses `NewFilter` because `tidal.New` builds a CRUD store). Import `go.rtnl.ai/tidal/filter` to call them under the `filter.` prefix instead. -WHERE operators: `Eq`, `Ne`, `Gt`, `Lt`, `Gte`, `Lte`, `Like`, `ILike`, `In`, `Is`, `IsNot`, `IsDistinctFrom`, and `IsNotDistinctFrom`. An `In` condition with an empty slice is omitted. `ILike`, `Is`, `IsNot`, `IsDistinctFrom`, and `IsNotDistinctFrom` are not supported by every database provider; invalid combinations fail at query time. +WHERE operators include `Eq`, `Ne`, `Gt`, `Lt`, `Gte`, `Lte`, `Like`, `ILike`, `In`, `NotIn`, `Is`, `IsNot`, `IsDistinctFrom`, and `IsNotDistinctFrom`. An empty `In` condition renders an always-false predicate; an empty `NotIn` condition renders an always-true predicate. Use `Any(op)` or `All(op)` with a comparison operator to build PostgreSQL-style quantified comparisons such as `= ANY (...)` or `> ALL (...)`; these are not supported by every provider. `ILike`, `Is`, `IsNot`, `IsDistinctFrom`, and `IsNotDistinctFrom` are also not supported by every database provider; invalid combinations fail at query time. -Use `Is` and `IsNot` with a [`Literal`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Literal) (`Null`, `True`, `False`, `Unknown`) for SQL keyword predicates such as `IS NULL` and `IS NOT TRUE`. Pass any other value to compare with a bound parameter (for example `status IS :w1`). PostgreSQL accepts `Literal` values only with `Is` and `IsNot`; SQLite also allows arbitrary bound values. `IsNull` and `IsNotNull` are deprecated; use `Is` with `Null` or `IsNot` with `Null` instead. +Use `Subselect` to insert a trusted SQL subquery into an `In`, `NotIn`, `Any`, or `All` condition. The subquery is rendered unchanged, and `Param` adds named parameters for placeholders inside it: + +```go +f := tidal.Where( + "experiment_id", + tidal.NotIn, + tidal.Subselect("SELECT experiment_id FROM task_versions WHERE task_id=:task_id") + ).Param("task_id", taskID) +// WHERE experiment_id NOT IN (SELECT experiment_id FROM task_versions WHERE task_id=:task_id) +``` + +For these operators, a string value beginning with `SELECT` or `WITH` is also +treated as a trusted subquery. Other string values remain bound parameters. + +Quantified comparisons bind an array as one parameter or accept a subquery: + +```go +f := tidal.Where("score", tidal.All(tidal.Gt), []int64{1, 2, 3}) +// WHERE score > ALL(:w1) +``` + +Use `Is` and `IsNot` with a [`Literal`](https://pkg.go.dev/go.rtnl.ai/tidal/filter#Literal) (`Null`, `True`, `False`, `Unknown`) for SQL keyword predicates such as `IS NULL` and `IS NOT TRUE`. Pass any other value to compare with a bound parameter (for example `status IS :w1`). PostgreSQL accepts `Literal` values only with `Is` and `IsNot`; SQLite also allows arbitrary bound values. `IsDistinctFrom` and `IsNotDistinctFrom` render null-safe comparisons with a bound parameter (for example `a IS DISTINCT FROM :w1`). Use `ILike` for case-insensitive pattern matching on providers that support it (for example PostgreSQL). @@ -176,11 +197,14 @@ f := tidal.Where("revoked", tidal.Is, tidal.Null). Flat `And`/`Or` chains follow SQL precedence: `Where("a", Eq, 1).And("b", Eq, 2).Or("c", Eq, 3)` renders `a = :w1 AND b = :w2 OR c = :w3`, which SQL parses as `(a AND b) OR c`. Use `AndGroup` or `OrGroup` for explicit grouping. ```go -// tidal.Where starts the filter; a second Where replaces the first WHERE clause. +// tidal.Where starts the filter; a second Where appends another AND condition. f := tidal.Where("status", tidal.Eq, "active"). - Where("role", tidal.Eq, "admin") // only role = :w1 remains + Where("role", tidal.Eq, "admin") // both conditions remain, joined by AND -// And/Or/AndGroup/OrGroup append to the current WHERE clause. +// And/Or/AndGroup/OrGroup append to the current WHERE clause. Inside an +// AndGroup or OrGroup callback, WhereGroup.Where also appends with AND, just +// like Filter.Where. Use WhereGroup.Set to replace that group's expression +// and WhereGroup.Reset to clear it. f = tidal.Where("status", tidal.Eq, "active"). And("id", tidal.In, []int64{1, 2, 3}). And("age", tidal.Gte, 18). @@ -290,13 +314,13 @@ fmt.Printf("schema at migration %d (%s), applied %s with %s\n", The `go.rtnl.ai/tidal/fields` package provides custom column types that implement the `database/sql` [`driver.Valuer`](https://pkg.go.dev/database/sql/driver#Valuer) and [`sql.Scanner`](https://pkg.go.dev/database/sql#Scanner) interfaces. This means they can be used directly as `Model` struct fields and passed to or scanned from the database without any extra conversion code. -| Field | Go type | Use for | Null handling | -| --- | --- | --- | --- | -| `JSONB` | `json.RawMessage` (`[]byte`) | Arbitrary JSON stored in a `JSONB` or `BYTEA` column | Empty/`null` JSON scans to a `nil` slice | -| `NullJSONB` | struct with `Valid bool` and `JSONB` | A nullable JSON column where you must distinguish SQL `NULL` from JSON `null`/`{}` | `Valid` is `false` when the column is `NULL` or the JSON is `null` | -| `StringArray` | `[]string` | A list of strings stored as a JSON array | Empty array scans to a `nil` slice | -| `NullStringArray` | struct with `Valid bool` and `StringArray` | A nullable list of strings | `Valid` is `false` when the column is `NULL` | -| `Timestamp` | struct wrapper around `time.Time` | UTC-normalized timestamp values with stable precision across drivers | Zero value writes/scans as SQL `NULL` | +| Field | Go type | Use for | Null handling | +| ----------------- | ------------------------------------------ | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `JSONB` | `json.RawMessage` (`[]byte`) | Arbitrary JSON stored in a `JSONB` or `BYTEA` column | Empty/`null` JSON scans to a `nil` slice | +| `NullJSONB` | struct with `Valid bool` and `JSONB` | A nullable JSON column where you must distinguish SQL `NULL` from JSON `null`/`{}` | `Valid` is `false` when the column is `NULL` or the JSON is `null` | +| `StringArray` | `[]string` | A list of strings stored as a JSON array | Empty array scans to a `nil` slice | +| `NullStringArray` | struct with `Valid bool` and `StringArray` | A nullable list of strings | `Valid` is `false` when the column is `NULL` | +| `Timestamp` | struct wrapper around `time.Time` | UTC-normalized timestamp values with stable precision across drivers | Zero value writes/scans as SQL `NULL` | `JSONB`, `NullJSONB`, `StringArray`, and `NullStringArray` marshal/scan as JSON, so those columns should use JSON-compatible storage (`JSONB` or `BYTEA` in Postgres, `BLOB`/`TEXT` in SQLite). `Timestamp` stores ISO-8601 UTC values and normalizes precision when read/written. @@ -583,11 +607,11 @@ go test ./bind -run '^$' -bench '^BenchmarkRewrite$' -benchmem -count=1 Last observed benchmark on darwin/arm64 (Apple M2): -| Case | ns/op | B/op | allocs/op | -| --- | ---: | ---: | ---: | -| OrderedSimple | 937.9 | 198 | 6 | -| OrderedComplex | 1407 | 308 | 5 | -| PositionalSimple | 549.4 | 256 | 4 | +| Case | ns/op | B/op | allocs/op | +| ---------------- | ----: | ---: | --------: | +| OrderedSimple | 937.9 | 198 | 6 | +| OrderedComplex | 1407 | 308 | 5 | +| PositionalSimple | 549.4 | 256 | 4 | SQLite tests need no setup. Each test suite creates its own database file in a temporary directory. diff --git a/filter/builder/builder.go b/filter/builder/builder.go index 8178753..d5a1772 100644 --- a/filter/builder/builder.go +++ b/filter/builder/builder.go @@ -1,10 +1,4 @@ // Package builder provides SQL clause fragments for [filter.Filter] list queries. -// -// Types such as [Where], [Ordering], [Limit], and [Offset] render ANSI SQL fragments -// (WHERE, ORDER BY, LIMIT, OFFSET). [Where] supports Eq, Ne, comparisons, Like, -// ILike, In, Is, IsNot (with [Literal] or a bound value), and IS DISTINCT FROM. -// Deprecated WhereOp values IsNull and IsNotNull remain supported. -// [filter.Filter] composes them into a single [filter.ListFilter] clause. package builder import ( diff --git a/filter/builder/in.go b/filter/builder/in.go index 7ba1648..52b1e5a 100644 --- a/filter/builder/in.go +++ b/filter/builder/in.go @@ -32,13 +32,7 @@ func inValues(value any) []any { } } -// Returns true if the condition should be added to the WHERE clause. -func shouldAddCondition(op WhereOp, value any) bool { - // All operators except In are always added. - if op != In { - return true - } - - // An In condition with an empty slice is omitted. - return len(inValues(value)) > 0 +// Reports whether a value is a nil or empty slice/array. +func isEmptySet(value any) bool { + return len(inValues(value)) == 0 } diff --git a/filter/builder/operator.go b/filter/builder/operator.go index 4b9f0f5..97ba58e 100644 --- a/filter/builder/operator.go +++ b/filter/builder/operator.go @@ -33,17 +33,12 @@ const ( // Not all database providers support it. ILike - // IsNull is the SQL 'IS NULL' operator for checking if a value is NULL. - // Deprecated: use [Is] with [Null] instead. - IsNull - - // IsNotNull is the SQL 'IS NOT NULL' operator for checking if a value is not NULL. - // Deprecated: use [IsNot] with [Null] instead. - IsNotNull - // In represents the SQL 'IN' operator for matching any value in a set/list. In + // NotIn represents the SQL 'NOT IN' operator for excluding values in a set/list. + NotIn + // Is is the SQL 'IS' operator. Use with a [Literal] for NULL and boolean predicates // (for example, field IS NULL), or with any other value for null-safe equality // (for example, field IS :param). Not all database providers support every form; @@ -77,8 +72,112 @@ const ( BitNot ) +const ( + quantifierAny WhereOp = 1 << 6 + quantifierAll WhereOp = 1 << 7 + quantifierMask = quantifierAny | quantifierAll + + anyEq = quantifierAny + anyNe = quantifierAny | Ne + anyGt = quantifierAny | Gt + anyLt = quantifierAny | Lt + anyGte = quantifierAny | Gte + anyLte = quantifierAny | Lte + anyLike = quantifierAny | Like + anyILike = quantifierAny | ILike + + allEq = quantifierAll + allNe = quantifierAll | Ne + allGt = quantifierAll | Gt + allLt = quantifierAll | Lt + allGte = quantifierAll | Gte + allLte = quantifierAll | Lte + allLike = quantifierAll | Like + allILike = quantifierAll | ILike +) + +// Builds an ANY comparison from a comparison operator. Unsupported operators +// are rendered as provided and may fail when the database executes the query. +func Any(op WhereOp) WhereOp { + switch op { + case Eq: + return anyEq + case Ne: + return anyNe + case Gt: + return anyGt + case Lt: + return anyLt + case Gte: + return anyGte + case Lte: + return anyLte + case Like: + return anyLike + case ILike: + return anyILike + default: + return quantifierAny | op + } +} + +// Builds an ALL comparison from a comparison operator. Unsupported operators +// are rendered as provided and may fail when the database executes the query. +func All(op WhereOp) WhereOp { + switch op { + case Eq: + return allEq + case Ne: + return allNe + case Gt: + return allGt + case Lt: + return allLt + case Gte: + return allGte + case Lte: + return allLte + case Like: + return allLike + case ILike: + return allILike + default: + return quantifierAll | op + } +} + +// Splits a quantified operator into its comparison and quantifier components. +func quantifiedOperator(op WhereOp) (WhereOp, string, bool) { + var quantifier string + switch op & quantifierMask { + case quantifierAny: + quantifier = "ANY" + case quantifierAll: + quantifier = "ALL" + case quantifierMask: + quantifier = "ANY ALL" + default: + return op, "", false + } + return op &^ quantifierMask, quantifier, true +} + +// Reports whether an operator is a standard scalar comparison operator. +func isComparisonOperator(op WhereOp) bool { + switch op { + case Eq, Ne, Gt, Lt, Gte, Lte, Like, ILike: + return true + default: + return false + } +} + // Returns the operator as a string for ANSI SQL. func (op WhereOp) String() string { + if comparison, quantifier, ok := quantifiedOperator(op); ok { + return comparison.String() + " " + quantifier + } + switch op { case Eq: return "=" @@ -96,14 +195,10 @@ func (op WhereOp) String() string { return "LIKE" case ILike: return "ILIKE" - case IsNull: - // Deprecated - return "IS NULL" - case IsNotNull: - // Deprecated - return "IS NOT NULL" case In: return "IN" + case NotIn: + return "NOT IN" case Is: return "IS" case IsNot: diff --git a/filter/builder/subquery.go b/filter/builder/subquery.go new file mode 100644 index 0000000..37eb919 --- /dev/null +++ b/filter/builder/subquery.go @@ -0,0 +1,31 @@ +package builder + +import "strings" + +// Subquery marks SQL that should be rendered as a subquery rather than bound as +// a string parameter. The SQL is inserted unchanged, so callers must ensure it +// is trusted and valid for the target database. Values passed directly as +// strings that begin with SELECT or WITH are recognized the same way. +type Subquery string + +// Builds a trusted subquery value for use with [In], [NotIn], [Any], or [All]. +func Subselect(query string) Subquery { + return Subquery(query) +} + +// Returns the SQL text when value represents a trusted subquery. +func subqueryValue(value any) (string, bool) { + switch query := value.(type) { + case Subquery: + return string(query), true + case string: + // Accept the concise ticket/API form while preserving ordinary scalar + // strings as bound values for IN and NOT IN. + trimmed := strings.TrimSpace(query) + upper := strings.ToUpper(trimmed) + if strings.HasPrefix(upper, "SELECT ") || strings.HasPrefix(upper, "WITH ") { + return trimmed, true + } + } + return "", false +} diff --git a/filter/builder/where.go b/filter/builder/where.go index 9a97f79..0259f36 100644 --- a/filter/builder/where.go +++ b/filter/builder/where.go @@ -3,6 +3,7 @@ package builder import ( "database/sql" "fmt" + "slices" "strings" ) @@ -13,74 +14,97 @@ import ( // Where builds a WHERE expression from conditions, logical operators, and groups. type Where struct { root *whereGroup + + // Parameters used by trusted SQL expressions such as subqueries. + params []sql.NamedArg } -// Reset clears any built WHERE expression. +// Clears the WHERE expression and any parameters attached to it. func (w *Where) Reset() { w.root = nil + w.params = nil } -// IsEmpty reports whether the WHERE expression has no conditions. +// Reports whether the WHERE expression has no conditions. func (w *Where) IsEmpty() bool { return w.root == nil || len(w.root.children) == 0 } -// Set replaces any existing expression with a single condition. +// Replaces any existing expression with a single condition. func (w *Where) Set(field string, op WhereOp, value any) { - if !shouldAddCondition(op, value) { - w.root = nil - return - } w.root = &whereGroup{ children: []whereNode{{node: &whereCondition{field: field, op: op, value: value}}}, } } -// Where replaces any existing expression with a single condition. +// Appends a condition joined with AND. func (w *Where) Where(field string, op WhereOp, value any) *Where { - w.Set(field, op, value) + w.And(field, op, value) return w } -// And appends a condition joined with AND. +// Appends a condition joined with AND. func (w *Where) And(field string, op WhereOp, value any) *Where { w.append(logAnd, field, op, value) return w } -// Or appends a condition joined with OR. +// Appends a condition joined with OR. func (w *Where) Or(field string, op WhereOp, value any) *Where { w.append(logOr, field, op, value) return w } -// AndGroup appends a parenthesized group joined with AND. +// Appends a parenthesized group joined with AND. func (w *Where) AndGroup(fn func(*Where)) { w.appendGroup(logAnd, fn) } -// OrGroup appends a parenthesized group joined with OR. +// Appends a parenthesized group joined with OR. func (w *Where) OrGroup(fn func(*Where)) { w.appendGroup(logOr, fn) } -// Render returns the WHERE clause and named parameters in traversal order. +// Adds or replaces a named parameter available to SQL expressions in the WHERE +// clause. +func (w *Where) Param(name string, value any) *Where { + for i := range w.params { + if w.params[i].Name == name { + w.params[i] = sql.Named(name, value) + return w + } + } + w.params = append(w.params, sql.Named(name, value)) + return w +} + +// Copies the WHERE expression and its named parameters. +func (w *Where) Clone() *Where { + if w == nil { + return nil + } + + return &Where{ + root: cloneWhereGroup(w.root), + params: slices.Clone(w.params), + } +} + +// Renders the WHERE clause and named parameters in traversal order. func (w *Where) Render() (string, []sql.NamedArg) { if w.IsEmpty() { - return "", nil + return "", slices.Clone(w.params) } var params []sql.NamedArg idx := 0 clause := w.root.render(¶ms, &idx) + params = append(params, w.params...) return "WHERE " + clause, params } // Appends a condition to the current WHERE clause. func (w *Where) append(logical logicalOp, field string, op WhereOp, value any) { - if !shouldAddCondition(op, value) { - return - } cond := &whereCondition{field: field, op: op, value: value} if w.IsEmpty() { w.root = &whereGroup{children: []whereNode{{node: cond}}} @@ -108,6 +132,32 @@ func (w *Where) appendGroup(logical logicalOp, fn func(*Where)) { w.root.children = append(w.root.children, whereNode{logical: logical, node: group}) } +// Copies a WHERE group and all of its expression nodes. +func cloneWhereGroup(group *whereGroup) *whereGroup { + if group == nil { + return nil + } + + clone := &whereGroup{ + children: make([]whereNode, len(group.children)), + grouped: group.grouped, + } + for i, child := range group.children { + clone.children[i].logical = child.logical + switch node := child.node.(type) { + case *whereCondition: + clone.children[i].node = &whereCondition{ + field: node.field, + op: node.op, + value: node.value, + } + case *whereGroup: + clone.children[i].node = cloneWhereGroup(node) + } + } + return clone +} + //============================================================================ // Expression model //============================================================================ @@ -168,16 +218,30 @@ func (g *whereGroup) render(params *[]sql.NamedArg, idx *int) string { func renderNode(node any, params *[]sql.NamedArg, idx *int) string { switch n := node.(type) { case *whereCondition: + // If we have a quantified operator (ALL/ANY), render it as a quantified + // condition like: "field op quantifier(values)" (ex: `foo != ANY(:w1)`). + if comparison, quantifier, ok := quantifiedOperator(n.op); ok { + return renderQuantified(n.field, comparison, quantifier, n.value, params, idx) + } + switch n.op { - case IsNull: - return fmt.Sprintf("%s IS NULL", n.field) - case IsNotNull: - return fmt.Sprintf("%s IS NOT NULL", n.field) - case In: + case In, NotIn: + // Render subqueries directly + if query, ok := subqueryValue(n.value); ok { + return fmt.Sprintf("%s %s (%s)", n.field, n.op, query) + } + + // If there are no values, return a condition that always evaluates + // to false/true vals := inValues(n.value) if len(vals) == 0 { - return "" + if n.op == In { + return "1=0" + } + return "1=1" } + + // Render the IN/NOT IN condition with placeholders placeholders := make([]string, len(vals)) for i, v := range vals { *idx++ @@ -185,12 +249,13 @@ func renderNode(node any, params *[]sql.NamedArg, idx *int) string { *params = append(*params, sql.Named(name, v)) placeholders[i] = ":" + name } - return fmt.Sprintf("%s IN (%s)", n.field, strings.Join(placeholders, ", ")) + return fmt.Sprintf("%s %s (%s)", n.field, n.op, strings.Join(placeholders, ", ")) case Is, IsNot: if sym, ok := n.value.(Literal); ok { + // Literal value like TRUE/FALSE/etc. return fmt.Sprintf("%s %s %s", n.field, n.op, sym) } - fallthrough + fallthrough // non-literal values render "field op value" default: // All other operators are rendered as: "field op value" *idx++ @@ -205,6 +270,31 @@ func renderNode(node any, params *[]sql.NamedArg, idx *int) string { } } +// Renders a quantified comparison against an array parameter or subquery. +func renderQuantified(field string, comparison WhereOp, quantifier string, value any, params *[]sql.NamedArg, idx *int) string { + if query, ok := subqueryValue(value); ok { + return fmt.Sprintf("%s %s %s (%s)", field, comparison, quantifier, query) + } + + if isComparisonOperator(comparison) && isEmptySet(value) { + // Supported quantifiers return their boolean identity above. Any other + // quantifier combination is invalid; leave it malformed so SQL validation + // fails instead of binding a made-up parameter. + if quantifier == "ANY" { + return "1=0" + } + if quantifier == "ALL" { + return "1=1" + } + return fmt.Sprintf("%s %s %s", field, comparison, quantifier) + } + + *idx++ + name := fmt.Sprintf("w%d", *idx) + *params = append(*params, sql.Named(name, value)) + return fmt.Sprintf("%s %s %s (:%s)", field, comparison, quantifier, name) +} + //============================================================================ // Field Prefixing for Joins //============================================================================ diff --git a/filter/builder/where_test.go b/filter/builder/where_test.go index 17197aa..80bbfe5 100644 --- a/filter/builder/where_test.go +++ b/filter/builder/where_test.go @@ -8,11 +8,17 @@ import ( "go.rtnl.ai/tidal/filter/builder" ) +// Check that table aliases are added to matching fields without changing the +// operators or parameter values. func TestWhere_Prefix(t *testing.T) { + // Make simple WHERE expressions for each alias case and compare the SQL and + // parameters after applying the alias. where := func(field string, op builder.WhereOp, value any) *builder.Where { return (&builder.Where{}).Where(field, op, value) } + // Cover adding, keeping, clearing, and replacing aliases, including an OR + // expression. testCases := []struct { orig *builder.Where alias string @@ -62,6 +68,8 @@ func TestWhere_Prefix(t *testing.T) { }, } + // Apply each alias and check that only the field names change and the + // parameters stay in the same order. for i, tc := range testCases { tc.orig.Prefix(tc.alias) actualClause, actualParams := tc.orig.Render() @@ -69,3 +77,121 @@ func TestWhere_Prefix(t *testing.T) { require.Equal(t, tc.expectedParams, actualParams, "test case %d failed", i) } } + +// Check conditions, lists, subqueries, ANY/ALL comparisons, and their +// parameters. +func TestWhere_Operators(t *testing.T) { + t.Run("WhereAppends", func(t *testing.T) { + // Add two conditions and check that the second is joined with AND instead + // of replacing the first. + where := (&builder.Where{}). + Where("status", builder.Eq, "active"). + Where("role", builder.Eq, "admin") + + clause, params := where.Render() + require.Equal(t, "WHERE status = :w1 AND role = :w2", clause) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", "active"), + sql.Named("w2", "admin"), + }, params) + }) + + t.Run("NotIn", func(t *testing.T) { + // Make a NOT IN list and check that each value gets a named placeholder + // in the same order. + where := (&builder.Where{}).Where("id", builder.NotIn, []int{1, 2, 3}) + + clause, params := where.Render() + require.Equal(t, "WHERE id NOT IN (:w1, :w2, :w3)", clause) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", 1), + sql.Named("w2", 2), + sql.Named("w3", 3), + }, params) + }) + + t.Run("EmptySetIdentities", func(t *testing.T) { + // Check that empty IN and NOT IN lists become false and true conditions + // instead of disappearing. + in := (&builder.Where{}).Where("id", builder.In, []int{}) + notIn := (&builder.Where{}).Where("id", builder.NotIn, []int{}) + + inClause, inParams := in.Render() + notInClause, notInParams := notIn.Render() + require.Equal(t, "WHERE 1=0", inClause) + require.Empty(t, inParams) + require.Equal(t, "WHERE 1=1", notInClause) + require.Empty(t, notInParams) + }) + + t.Run("SubselectAndParameter", func(t *testing.T) { + // Use a trusted subquery with a named parameter and check that the query + // is kept as SQL instead of being bound as a string. + where := (&builder.Where{}). + Where("id", builder.NotIn, builder.Subselect("SELECT id FROM deleted WHERE owner=:owner")). + Param("owner", "ada") + + clause, params := where.Render() + require.Equal(t, "WHERE id NOT IN (SELECT id FROM deleted WHERE owner=:owner)", clause) + require.Equal(t, []sql.NamedArg{sql.Named("owner", "ada")}, params) + }) + + t.Run("AnySubselect", func(t *testing.T) { + // Check that an ANY comparison keeps its operator and trusted subquery. + where := (&builder.Where{}). + Where("id", builder.Any(builder.Eq), builder.Subselect("SELECT id FROM candidates")) + + clause, params := where.Render() + require.Equal(t, "WHERE id = ANY (SELECT id FROM candidates)", clause) + require.Empty(t, params) + }) + + t.Run("WithSubselectString", func(t *testing.T) { + // Check that a WITH query string is treated as a subquery instead of a + // string parameter. + where := (&builder.Where{}). + Where("id", builder.Any(builder.Eq), "WITH candidates AS (SELECT id FROM source) SELECT id FROM candidates") + + clause, params := where.Render() + require.Equal(t, "WHERE id = ANY (WITH candidates AS (SELECT id FROM source) SELECT id FROM candidates)", clause) + require.Empty(t, params) + }) + + t.Run("AllArray", func(t *testing.T) { + // Check that an ALL comparison keeps the whole slice as one array + // parameter. + where := (&builder.Where{}).Where("id", builder.All(builder.Gt), []int64{1, 2}) + + clause, params := where.Render() + require.Equal(t, "WHERE id > ALL (:w1)", clause) + require.Equal(t, []sql.NamedArg{sql.Named("w1", []int64{1, 2})}, params) + }) + + t.Run("QuantifiedEmptySetIdentities", func(t *testing.T) { + // Check the empty-set rules: ANY is false and ALL is true. The condition + // must remain in the query instead of disappearing. + any := (&builder.Where{}).Where("id", builder.Any(builder.Eq), []int{}) + all := (&builder.Where{}).Where("id", builder.All(builder.Eq), []int{}) + + anyClause, anyParams := any.Render() + allClause, allParams := all.Render() + require.Equal(t, "WHERE 1=0", anyClause) + require.Empty(t, anyParams) + require.Equal(t, "WHERE 1=1", allClause) + require.Empty(t, allParams) + }) + + t.Run("QuantifiedComparisonOperators", func(t *testing.T) { + // Check that Any and All keep the chosen comparison operator and bind + // each slice as one parameter. + any := (&builder.Where{}).Where("name", builder.Any(builder.Ne), []string{"ada", "grace"}) + all := (&builder.Where{}).Where("score", builder.All(builder.Lte), []int64{10, 20}) + + anyClause, anyParams := any.Render() + allClause, allParams := all.Render() + require.Equal(t, "WHERE name != ANY (:w1)", anyClause) + require.Equal(t, []sql.NamedArg{sql.Named("w1", []string{"ada", "grace"})}, anyParams) + require.Equal(t, "WHERE score <= ALL (:w1)", allClause) + require.Equal(t, []sql.NamedArg{sql.Named("w1", []int64{10, 20})}, allParams) + }) +} diff --git a/filter/filter.go b/filter/filter.go index 7c87dbb..5bea3a9 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -3,11 +3,18 @@ // Use [Filter] for WHERE, ORDER BY, LIMIT, and OFFSET. Use [CustomFilter] when you // need hand-written SQL (for example GROUP BY). // +// Use [Subselect] with [In], [NotIn], [Any], or [All] when a trusted SQL subquery is +// needed. Provider-specific SQL is passed through for the database to validate. +// [Filter.Where] and [WhereGroup.Where] append with AND rather than silently +// replacing the WHERE filter; use [Filter.ReplaceWhere] to replace the WHERE +// expression and [Filter.ClearWhere] to clear it. [Filter.Clear] clears every +// filter clause. +// // Building a composable [Filter] (can use tidal aliases): // // f := (&tidal.Filter{}). // Where("status", tidal.Eq, "active"). -// AndGroup(func(g *tidal.Where) { +// AndGroup(func(g *tidal.WhereGroup) { // g.Where("role", tidal.Eq, "admin"). // Or("role", tidal.Eq, "editor"). // Or("role", tidal.Eq, "viewer") @@ -30,6 +37,7 @@ package filter import ( "database/sql" + "slices" "strings" "go.rtnl.ai/tidal/filter/builder" @@ -50,7 +58,10 @@ type ListFilter interface { Params() []sql.NamedArg } -// WhereGroup builds a grouped WHERE expression inside [Filter.AndGroup] and [Filter.OrGroup]. +// WhereGroup builds a grouped WHERE expression inside [Filter.AndGroup] and +// [Filter.OrGroup]. Its Where method appends with AND, just like +// [Filter.Where]. Use [Set] to replace the group's expression and [Reset] to clear +// it. type WhereGroup = builder.Where // WhereOp is a comparison operator in a WHERE condition. @@ -60,6 +71,9 @@ type WhereOp = builder.WhereOp // predicates. type Literal = builder.Literal +// Subquery is trusted SQL rendered inside IN, NOT IN, ANY, or ALL expressions. +type Subquery = builder.Subquery + // SQL literals for [Is] and [IsNot] predicates. const ( Null = builder.Null @@ -78,9 +92,8 @@ const ( Lte = builder.Lte Like = builder.Like ILike = builder.ILike - IsNull = builder.IsNull // Deprecated: use [Is] with [Null] instead. - IsNotNull = builder.IsNotNull // Deprecated: use [IsNot] with [Null] instead. In = builder.In + NotIn = builder.NotIn Is = builder.Is IsNot = builder.IsNot IsDistinctFrom = builder.IsDistinctFrom @@ -91,6 +104,24 @@ const ( BitNot = builder.BitNot ) +// Builds an ANY comparison from a comparison operator. Unsupported operators +// are rendered as provided and may fail when the database executes the query. +func Any(op WhereOp) WhereOp { + // TODO: add doc comment from filter.Any(op) above anytime it is + // updated; Go will pass through docs for type aliases but not for + // function redirects + return builder.Any(op) +} + +// Builds an ALL comparison from a comparison operator. Unsupported operators +// are rendered as provided and may fail when the database executes the query. +func All(op WhereOp) WhereOp { + // TODO: add doc comment from filter.All(op) above anytime it is + // updated; Go will pass through docs for type aliases but not for + // function redirects + return builder.All(op) +} + //============================================================================ // Filter Implementation //============================================================================ @@ -128,6 +159,14 @@ func New() *Filter { return &Filter{} } +// Builds a trusted SQL subquery for use with [In], [NotIn], [Any], or [All]. +func Subselect(query string) Subquery { + // TODO: add doc comment from filter.Subselect(query) above anytime it is + // updated; Go will pass through docs for type aliases but not for + // function redirects + return builder.Subselect(query) +} + // Create a new filter with a WHERE clause. func Where(field string, op WhereOp, value any) *Filter { return New().Where(field, op, value) @@ -152,12 +191,74 @@ func Offset(n int) *Filter { // Where Building Methods //============================================================================ -// Calling Where replaces any previously built WHERE clause and starts a new one. +// Appends a condition joined with AND. For In, NotIn, Any, and All, a [Subquery] +// value or a string beginning with SELECT or WITH is rendered as trusted SQL. +// Use [Filter.ReplaceWhere] to intentionally discard the existing WHERE +// expression. func (f *Filter) Where(field string, op WhereOp, value any) *Filter { - f.ensureWhere().Set(field, op, value) + f.ensureWhere().And(field, op, value) + return f +} + +// Replaces the existing WHERE expression with a single condition. +func (f *Filter) ReplaceWhere(field string, op WhereOp, value any) *Filter { + f.resetCache() + if f.whereClause == nil { + f.whereClause = &builder.Where{} + } else { + f.whereClause.Reset() + } + f.whereClause.Set(field, op, value) + return f +} + +// Adds or replaces a named parameter available to SQL expressions in the WHERE +// clause. +func (f *Filter) Param(name string, value any) *Filter { + f.ensureWhere().Param(name, value) return f } +// Clears the WHERE expression and its parameters while retaining other clauses. +func (f *Filter) ClearWhere() *Filter { + f.resetCache() + if f.whereClause != nil { + f.whereClause.Reset() + } + return f +} + +// Clears the Filter completely. +func (f *Filter) Clear() *Filter { + f.resetCache() + f.whereClause = nil + f.ordering = nil + f.limit = nil + f.offset = nil + return f +} + +// Copies the filter so subsequent mutations do not affect the original. +func (f *Filter) Clone() *Filter { + if f == nil { + return nil + } + + clone := &Filter{ + whereClause: f.whereClause.Clone(), + ordering: slices.Clone(f.ordering), + } + if f.limit != nil { + limit := *f.limit + clone.limit = &limit + } + if f.offset != nil { + offset := *f.offset + clone.offset = &offset + } + return clone +} + // And appends a condition joined with AND. // // To control grouping explicitly, use [Filter.AndGroup]. @@ -306,6 +407,7 @@ func (f *Filter) Offset(n int) *Filter { // Prefixes all fields in the filter with a table alias. func (f *Filter) Prefix(tableAlias string, fields ...string) *Filter { + f.resetCache() if f.whereClause != nil { f.whereClause.Prefix(tableAlias, fields...) } diff --git a/filter/filter_test.go b/filter/filter_test.go index aae376f..d84f9ee 100644 --- a/filter/filter_test.go +++ b/filter/filter_test.go @@ -11,6 +11,7 @@ import ( // ORDER BY Clause Builder //============================================================================ +// Check ascending and descending order, including replacing an old order. func TestFilterOrder(t *testing.T) { t.Run("OrderByAscDesc", func(t *testing.T) { // Bare column is ASC; -prefix is DESC. @@ -29,13 +30,17 @@ func TestFilterOrder(t *testing.T) { // LIMIT / OFFSET Clause Builder //============================================================================ +// Check limit and offset, including removing them with negative values. func TestFilterLimitOffset(t *testing.T) { t.Run("LimitOffset", func(t *testing.T) { + // Set an order, limit, and offset, then check the full SQL clause. f := (&Filter{}).OrderBy("id").Limit(10).Offset(5) require.Equal(t, "ORDER BY id ASC LIMIT 10 OFFSET 5", f.Clause()) }) t.Run("LimitOffsetWithoutOrder", func(t *testing.T) { + // Set a limit and offset without an order, then check they work on their + // own. f := (&Filter{}).Limit(10).Offset(5) require.Equal(t, "LIMIT 10 OFFSET 5", f.Clause()) }) @@ -51,17 +56,24 @@ func TestFilterLimitOffset(t *testing.T) { // WHERE Clause Builder //============================================================================ +// Check conditions, groups, operators, subqueries, filter changes, and +// parameter order. func TestFilterWhere(t *testing.T) { - t.Run("WhereOverwrite", func(t *testing.T) { + t.Run("WhereAppends", func(t *testing.T) { + // Add two conditions and check that both remain with parameters in order. f := (&Filter{}). Where("status", Eq, "active"). Where("role", Eq, "admin") - require.Equal(t, "WHERE role = :w1", f.Clause()) - require.Equal(t, []sql.NamedArg{sql.Named("w1", "admin")}, f.Params()) + require.Equal(t, "WHERE status = :w1 AND role = :w2", f.Clause()) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", "active"), + sql.Named("w2", "admin"), + }, f.Params()) }) t.Run("WhereAndOrAppend", func(t *testing.T) { + // Add AND and OR conditions and check their order and parameters. f := (&Filter{}). Where("status", Eq, "active"). And("age", Gte, 18). @@ -76,6 +88,8 @@ func TestFilterWhere(t *testing.T) { }) t.Run("AndGroup", func(t *testing.T) { + // Add an AND group with AND and OR terms, then check its parentheses and + // parameter order. f := (&Filter{}). Where("status", Eq, "active"). AndGroup(func(g *WhereGroup) { @@ -91,6 +105,7 @@ func TestFilterWhere(t *testing.T) { }) t.Run("OrGroup", func(t *testing.T) { + // Add an OR group and check that it has its own parentheses. f := (&Filter{}). Where("status", Eq, "active"). OrGroup(func(g *WhereGroup) { @@ -101,6 +116,7 @@ func TestFilterWhere(t *testing.T) { }) t.Run("WhereWithOrderLimitOffset", func(t *testing.T) { + // Set a condition, order, limit, and offset, then check their SQL order. f := (&Filter{}). Where("status", Eq, "active"). OrderBy("-created"). @@ -111,6 +127,7 @@ func TestFilterWhere(t *testing.T) { }) t.Run("LikeOperator", func(t *testing.T) { + // Add two LIKE conditions and check that both patterns are parameters. f := (&Filter{}). Where("name", Like, "%ada%"). And("email", Like, "%@example.com") @@ -119,6 +136,8 @@ func TestFilterWhere(t *testing.T) { }) t.Run("ILikeOperator", func(t *testing.T) { + // Add a case-insensitive LIKE condition and check its operator and + // pattern parameter. f := (&Filter{}).Where("name", ILike, "%ada%") require.Equal(t, "WHERE name ILIKE :w1", f.Clause()) @@ -126,6 +145,8 @@ func TestFilterWhere(t *testing.T) { }) t.Run("InOperator", func(t *testing.T) { + // Add one normal condition and an IN list, then check all values get + // placeholders in order. f := (&Filter{}). Where("status", Eq, "active"). And("id", In, []int64{1, 2, 3}) @@ -139,23 +160,97 @@ func TestFilterWhere(t *testing.T) { }, f.Params()) }) - t.Run("InEmptySliceOmitted", func(t *testing.T) { + t.Run("InEmptySliceFalse", func(t *testing.T) { + // Add an empty IN list and check it becomes false instead of removing + // the filter. f := (&Filter{}). Where("status", Eq, "active"). And("id", In, []string{}) - require.Equal(t, "WHERE status = :w1", f.Clause()) + require.Equal(t, "WHERE status = :w1 AND 1=0", f.Clause()) require.Equal(t, []sql.NamedArg{sql.Named("w1", "active")}, f.Params()) }) - t.Run("InEmptySliceOnly", func(t *testing.T) { - f := (&Filter{}).Where("id", In, []int{}) + t.Run("NotInAnyAndAll", func(t *testing.T) { + // Add a NOT IN list and ANY/ALL conditions, then check the SQL and + // parameters for each one. + f := (&Filter{}). + Where("id", NotIn, []int64{1, 2}). + And("status", Any(Eq), Subselect("SELECT status FROM archived_statuses")). + And("priority", All(Gt), []int64{1, 2}) + + require.Equal(t, "WHERE id NOT IN (:w1, :w2) AND status = ANY (SELECT status FROM archived_statuses) AND priority > ALL (:w3)", f.Clause()) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", int64(1)), + sql.Named("w2", int64(2)), + sql.Named("w3", []int64{1, 2}), + }, f.Params()) + }) - require.Empty(t, f.Clause()) - require.Empty(t, f.Params()) + t.Run("SubselectParameters", func(t *testing.T) { + // Use a SELECT string with a named parameter and check that the SELECT + // stays as SQL while its value is bound separately. + f := (&Filter{}). + Where("experiment_id", NotIn, "SELECT experiment_id FROM task_versions WHERE task_id=:task_id"). + Param("task_id", 42) + + require.Equal(t, "WHERE experiment_id NOT IN (SELECT experiment_id FROM task_versions WHERE task_id=:task_id)", f.Clause()) + require.Equal(t, []sql.NamedArg{sql.Named("task_id", 42)}, f.Params()) + }) + + t.Run("CloneAndClearWhere", func(t *testing.T) { + // Make and clone a filter, change the clone, and check that the original + // is unchanged and the clone keeps its order and limit after clearing + // WHERE. + base := (&Filter{}). + Where("tenant_id", Eq, 7). + OrderBy("-created"). + Limit(10) + clone := base.Clone(). + Where("status", Eq, "active"). + Prefix("t") + + require.Equal(t, "WHERE tenant_id = :w1 ORDER BY created DESC LIMIT 10", base.Clause()) + require.Equal(t, "WHERE t.tenant_id = :w1 AND t.status = :w2 ORDER BY t.created DESC LIMIT 10", clone.Clause()) + + clone.ClearWhere() + require.Equal(t, "ORDER BY t.created DESC LIMIT 10", clone.Clause()) + }) + + t.Run("CloneDeepCopiesGroupsAndParameters", func(t *testing.T) { + // Make and cache a filter with a nested group and named parameter, then + // change the clone. The original must keep its fields and value. + base := (&Filter{}). + Where("tenant_id", Eq, 7). + AndGroup(func(g *WhereGroup) { + g.Where("status", Eq, "active").Or("status", Eq, "pending") + }). + Where("id", In, Subselect("SELECT id FROM memberships WHERE owner=:owner")). + Param("owner", "ada") + _ = base.Clause() + _ = base.Params() + + clone := base.Clone().Prefix("t").Param("owner", "grace") + + require.Equal(t, "WHERE tenant_id = :w1 AND (status = :w2 OR status = :w3) AND id IN (SELECT id FROM memberships WHERE owner=:owner)", base.Clause()) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", 7), + sql.Named("w2", "active"), + sql.Named("w3", "pending"), + sql.Named("owner", "ada"), + }, base.Params()) + require.Equal(t, "WHERE t.tenant_id = :w1 AND (t.status = :w2 OR t.status = :w3) AND t.id IN (SELECT id FROM memberships WHERE owner=:owner)", clone.Clause()) + require.Equal(t, []sql.NamedArg{ + sql.Named("w1", 7), + sql.Named("w2", "active"), + sql.Named("w3", "pending"), + sql.Named("owner", "grace"), + }, clone.Params()) }) t.Run("IsIsNotLiterals", func(t *testing.T) { + // Use SQL literals and check that they appear directly without + // parameters. f := (&Filter{}). Where("revoked", Is, Null). And("deleted_at", IsNot, Null). @@ -168,6 +263,8 @@ func TestFilterWhere(t *testing.T) { }) t.Run("IsWithValue", func(t *testing.T) { + // Use normal values with IS and IS NOT and check that they use + // parameters. f := (&Filter{}). Where("status", Is, "active"). And("legacy", IsNot, true) @@ -180,6 +277,8 @@ func TestFilterWhere(t *testing.T) { }) t.Run("IsDistinctFromOperators", func(t *testing.T) { + // Use both distinctness operators and check that their values are + // bound in order, including nil. f := (&Filter{}). Where("a", IsDistinctFrom, nil). And("b", IsNotDistinctFrom, "active") @@ -191,37 +290,30 @@ func TestFilterWhere(t *testing.T) { }, f.Params()) }) - t.Run("DeprecatedNullOperators", func(t *testing.T) { - f := (&Filter{}). - Where("revoked", IsNull, nil). - And("deleted_at", IsNotNull, nil) - - require.Equal(t, "WHERE revoked IS NULL AND deleted_at IS NOT NULL", f.Clause()) - require.Empty(t, f.Params()) - }) } //============================================================================ // WHERE Misuse / Edge Cases //============================================================================ -// Documents current behavior when WHERE helpers are called out of the usual order. -// [Filter] does not return errors today; these tests lock in what happens instead. +// Check what happens when WHERE methods are called in unusual orders. +// Filter does not return errors, so these cases define the current behavior. func TestFilterWhereMisuse(t *testing.T) { t.Run("OrBeforeWhereStartsFirstCondition", func(t *testing.T) { - // Or on an empty filter acts like the first condition; no OR keyword is emitted. + // OR on an empty filter becomes the first condition without an OR keyword. f := (&Filter{}).Or("role", Eq, "admin") require.Equal(t, "WHERE role = :w1", f.Clause()) }) t.Run("AndBeforeWhereStartsFirstCondition", func(t *testing.T) { - // And on an empty filter acts like the first condition; no AND keyword is emitted. + // AND on an empty filter becomes the first condition without an AND keyword. f := (&Filter{}).And("age", Gte, 18) require.Equal(t, "WHERE age >= :w1", f.Clause()) }) t.Run("OrGroupBeforeWhereBecomesRootGroup", func(t *testing.T) { - // An empty root accepts the group directly; no leading OR is emitted. + // An OR group on an empty filter becomes the first group without an OR + // keyword. f := (&Filter{}).OrGroup(func(g *WhereGroup) { g.Where("role", Eq, "admin") }) @@ -229,33 +321,36 @@ func TestFilterWhereMisuse(t *testing.T) { }) t.Run("AndGroupBeforeWhereBecomesRootGroup", func(t *testing.T) { - // An empty root accepts the group directly; no leading AND is emitted. + // An AND group on an empty filter becomes the first group without an AND + // keyword. f := (&Filter{}).AndGroup(func(g *WhereGroup) { g.Where("role", Eq, "admin") }) require.Equal(t, "WHERE (role = :w1)", f.Clause()) }) - t.Run("OrThenWhereReplacesPriorCondition", func(t *testing.T) { - // Where replaces the entire clause + t.Run("OrThenWhereAppends", func(t *testing.T) { + // WHERE still adds an AND condition after OR; it does not replace the + // existing condition. f := (&Filter{}). Or("legacy", Eq, true). Where("status", Eq, "active") - require.Equal(t, "WHERE status = :w1", f.Clause()) + require.Equal(t, "WHERE legacy = :w1 AND status = :w2", f.Clause()) }) - t.Run("OrGroupThenWhereReplacesEntireClause", func(t *testing.T) { - // Where replaces the entire clause + t.Run("OrGroupThenWhereAppends", func(t *testing.T) { + // WHERE adds an AND condition after an OR group; replacing it requires + // an explicit call to ReplaceWhere. f := (&Filter{}). OrGroup(func(g *WhereGroup) { g.Where("role", Eq, "admin") }). Where("status", Eq, "active") - require.Equal(t, "WHERE status = :w1", f.Clause()) + require.Equal(t, "WHERE (role = :w1 AND status = :w2)", f.Clause()) }) t.Run("EmptyOrGroupThenWhere", func(t *testing.T) { - // Where replaces the entire clause + // An empty group is ignored, so WHERE becomes the first condition. f := (&Filter{}). OrGroup(func(g *WhereGroup) {}). Where("status", Eq, "active") @@ -263,14 +358,14 @@ func TestFilterWhereMisuse(t *testing.T) { }) t.Run("EmptyOrGroupOnly", func(t *testing.T) { - // An empty group is ignored; no OR keyword is emitted. + // An empty group is ignored and produces no SQL. f := (&Filter{}).OrGroup(func(g *WhereGroup) {}) require.Empty(t, f.Clause()) require.Empty(t, f.Params()) }) t.Run("OrOrWithoutWhere", func(t *testing.T) { - // First Or starts the expression; the second Or appends normally. + // The first OR starts the filter; the second OR adds another condition. f := (&Filter{}). Or("a", Eq, 1). Or("b", Eq, 2) @@ -278,22 +373,68 @@ func TestFilterWhereMisuse(t *testing.T) { }) t.Run("OrGroupOrBeforeWhereInGroup", func(t *testing.T) { - // Inside a group, Or starts the sub-expression; Where replaces it. + // Inside a group, OR starts the group and WHERE adds an AND condition, + // just as it does on Filter. f := (&Filter{}).OrGroup(func(g *WhereGroup) { g.Or("role", Eq, "admin").Where("status", Eq, "active") }) - require.Equal(t, "WHERE (status = :w1)", f.Clause()) + require.Equal(t, "WHERE (role = :w1 AND status = :w2)", f.Clause()) }) - t.Run("OrOrGroupThenWhereReplacesAll", func(t *testing.T) { - // Where replaces the entire clause + t.Run("OrOrGroupThenWhereAppends", func(t *testing.T) { + // WHERE adds an AND condition after existing OR terms; use ReplaceWhere + // when the old filter should be discarded. f := (&Filter{}). Or("legacy", Eq, true). OrGroup(func(g *WhereGroup) { g.Where("role", Eq, "admin") }). Where("status", Eq, "active") - require.Equal(t, "WHERE status = :w1", f.Clause()) + require.Equal(t, "WHERE legacy = :w1 OR (role = :w2) AND status = :w3", f.Clause()) + }) + + t.Run("ReplaceWhere", func(t *testing.T) { + // Cache a filter with a subquery parameter and pagination, replace WHERE, + // and check that pagination remains while the old condition and parameter + // are removed. + f := (&Filter{}). + Where("legacy_id", NotIn, Subselect("SELECT id FROM legacy WHERE owner=:old_owner")). + Param("old_owner", "ada"). + OrderBy("-created"). + Limit(20). + Offset(5) + _ = f.Clause() + _ = f.Params() + + f.ReplaceWhere("status", Eq, "active") + + require.Equal(t, "WHERE status = :w1 ORDER BY created DESC LIMIT 20 OFFSET 5", f.Clause()) + require.Equal(t, []sql.NamedArg{sql.Named("w1", "active")}, f.Params()) + }) + + t.Run("ClearWhereAndClear", func(t *testing.T) { + // Cache a filter with every clause, clear WHERE, and check that order, + // limit, and offset remain while WHERE parameters are removed. + f := (&Filter{}). + Where("id", In, Subselect("SELECT id FROM records WHERE owner=:owner")). + Param("owner", "ada"). + OrderBy("id"). + Limit(10). + Offset(5) + _ = f.Clause() + _ = f.Params() + + f.ClearWhere() + require.Equal(t, "ORDER BY id ASC LIMIT 10 OFFSET 5", f.Clause()) + require.Empty(t, f.Params()) + + // Clear changes the filter itself, so callers do not need to assign its + // return value. + cleared := f.Where("status", Eq, "active").Clear() + require.Same(t, f, cleared) + require.Equal(t, &Filter{}, f) + require.Empty(t, f.Clause()) + require.Empty(t, f.Params()) }) } @@ -301,8 +442,7 @@ func TestFilterWhereMisuse(t *testing.T) { // Comprehensive Clause Builder //============================================================================ -// Comprehensive test of all WHERE, ORDER BY, LIMIT, and OFFSET building -// methods with a really, really, REALLY complex filter. +// Check a filter that combines WHERE, ORDER BY, LIMIT, and OFFSET. func TestFilterComprehensive(t *testing.T) { f := (&Filter{}). Where("status", Eq, "active"). @@ -432,15 +572,23 @@ func TestFilterComprehensive(t *testing.T) { }, f.Params()) } +// Verify prefixing updates both WHERE fields and ORDER BY columns after the +// filter has already rendered and cached its output. func TestFilterPrefix(t *testing.T) { + // Build and render a filter first so this test also exercises cache + // invalidation when prefixing is applied afterward. f := New(). Where("color", Eq, "red"). And("age", Gte, 25). OrderBy("-age", "created"). Limit(10). - Offset(30). - Prefix("t") + Offset(30) + _ = f.Clause() + _ = f.Params() + f.Prefix("t") + // Assert that both WHERE fields and ORDER BY columns receive the prefix, + // while pagination and parameter numbering remain unchanged. require.Equal(t, "WHERE t.color = :w1 AND t.age >= :w2 ORDER BY t.age DESC, t.created ASC LIMIT 10 OFFSET 30", f.Clause(), diff --git a/filter/postgres_test.go b/filter/postgres_test.go new file mode 100644 index 0000000..64466a2 --- /dev/null +++ b/filter/postgres_test.go @@ -0,0 +1,85 @@ +package filter_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.rtnl.ai/tidal/filter" + "go.rtnl.ai/tidal/suite" +) + +// Exercises ANY and ALL array parameters through the configured PostgreSQL +// database/sql driver and verifies the rows returned by each comparison. +type filterPostgresSuite struct { + suite.PostgresSuite +} + +// Exercises ANY and ALL array parameters through the configured PostgreSQL +// database/sql driver and verifies the rows returned by each comparison. +func TestPostgresFilter(t *testing.T) { + s := &filterPostgresSuite{} + suite.Run(t, s) +} + +func (s *filterPostgresSuite) TestAnyAndAllArrayParameters() { + // Set up a temporary table with scalar BIGINT columns. The scalar columns + // let PostgreSQL compare each row against the array parameter supplied to + // ANY or ALL. + _, err := s.SQLDB().ExecContext(s.Context(), ` + CREATE TEMPORARY TABLE filter_quantified_values ( + id BIGINT PRIMARY KEY, + score BIGINT NOT NULL + )`) + require.NoError(s.T(), err) + + // Insert predictable IDs and scores so the expected results for equality + // and greater-than quantified comparisons are unambiguous. + _, err = s.SQLDB().ExecContext(s.Context(), ` + INSERT INTO filter_quantified_values (id, score) + VALUES (1, 10), (2, 20), (3, 30)`) + require.NoError(s.T(), err) + + // Use a real Tidal transaction for both queries. Passing the filter's + // named arguments through tx.Query exercises placeholder rewriting and + // pgx's conversion of []int64 into a PostgreSQL array. + tx := s.BeginTx(nil) + defer tx.Rollback() + + // Match IDs equal to any element in [1, 3]. The assertion below expects + // PostgreSQL to return only the first and third rows. + anyFilter := filter.Where("id", filter.Any(filter.Eq), []int64{1, 3}) + anyRows, err := tx.Query( + "SELECT id FROM filter_quantified_values "+anyFilter.Clause()+" ORDER BY id", + anyFilter.Params()..., + ) + require.NoError(s.T(), err) + defer anyRows.Close() + + var anyIDs []int64 + for anyRows.Next() { + var id int64 + require.NoError(s.T(), anyRows.Scan(&id)) + anyIDs = append(anyIDs, id) + } + require.NoError(s.T(), anyRows.Err()) + require.Equal(s.T(), []int64{1, 3}, anyIDs) + + // Match scores greater than every element in [1, 2]. Since all inserted + // scores satisfy that comparison, the assertion expects all three IDs. + allFilter := filter.Where("score", filter.All(filter.Gt), []int64{1, 2}) + allRows, err := tx.Query( + "SELECT id FROM filter_quantified_values "+allFilter.Clause()+" ORDER BY id", + allFilter.Params()..., + ) + require.NoError(s.T(), err) + defer allRows.Close() + + var allIDs []int64 + for allRows.Next() { + var id int64 + require.NoError(s.T(), allRows.Scan(&id)) + allIDs = append(allIDs, id) + } + require.NoError(s.T(), allRows.Err()) + require.Equal(s.T(), []int64{1, 2, 3}, allIDs) +} diff --git a/tidal.go b/tidal.go index 33f2d48..f14c75f 100644 --- a/tidal.go +++ b/tidal.go @@ -65,7 +65,10 @@ type ( ) var ( + // Open opens a database connection for the provided DSN. Open = conn.Open + + // Wrap wraps an existing database connection. Wrap = conn.Wrap ) @@ -113,21 +116,33 @@ type ( CustomFilter = filter.CustomFilter ListFilter = filter.ListFilter WhereGroup = filter.WhereGroup + WhereOp = filter.WhereOp + Subquery = filter.Subquery ) var ( - // Filter constructors - + // NewFilter creates an empty filter. NewFilter = filter.New - Where = filter.Where - OrderBy = filter.OrderBy - Limit = filter.Limit - Offset = filter.Offset + + // Where creates a filter with an initial WHERE condition. + Where = filter.Where + + // Subselect builds a trusted SQL subquery for use with In, NotIn, Any, or All. + Subselect = filter.Subselect + + // OrderBy creates a filter with an ORDER BY clause. + OrderBy = filter.OrderBy + + // Limit creates a filter with a LIMIT clause. + Limit = filter.Limit + + // Offset creates a filter with an OFFSET clause. + Offset = filter.Offset ) -const ( - // Where Operations +// Where Operations +const ( Eq = builder.Eq Ne = builder.Ne Gt = builder.Gt @@ -136,9 +151,8 @@ const ( Lte = builder.Lte Like = builder.Like ILike = builder.ILike - IsNull = builder.IsNull // Deprecated: use [Is] with [Null] instead. - IsNotNull = builder.IsNotNull // Deprecated: use [IsNot] with [Null] instead. In = builder.In + NotIn = builder.NotIn Is = builder.Is IsNot = builder.IsNot IsDistinctFrom = builder.IsDistinctFrom @@ -157,6 +171,18 @@ const ( OrderDESC = builder.OrderDESC ) +var ( + // Any builds an ANY comparison from a comparison operator. Unsupported + // operators are rendered as provided and may fail when the database executes + // the query. + Any = filter.Any + + // All builds an ALL comparison from a comparison operator. Unsupported + // operators are rendered as provided and may fail when the database executes + // the query. + All = filter.All +) + // Store type (