Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions changelog.d/rp-847-series-scan-cost.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
### EXPLAIN prices a filtered generate_series scan

An `EXPLAIN` of a `generate_series` in FROM with a WHERE now carries
PostgreSQL's estimate instead of `rows=0 width=0`. As `cost_functionscan`
does, every row the series yields pays the tuple cost and the Filter's
operators, and the row estimate is the series' rows times the WHERE's
selectivity over a relation with no statistics: an equality is one over
the series' rows (up to 200 distinct), an inequality a third, a `<` and
`>` pair on the column 0.005, and an OR combines its arms.

#### Fixed

- `explain select 1 from generate_series(1, 2) f(x) where x > 1` prints
`(cost=0.00..0.03 rows=1 width=4)`, as PostgreSQL 15.19 does.
Fixes RP-847 (tests/backlog/test_rp_found_rp703.py::test_a_filtered_series_scan_is_priced)
11 changes: 11 additions & 0 deletions changelog.d/rp-866-out-of-range-position.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
### A literal's out-of-range error points at the literal

`select '70000'::int2` now reports its 22003, `value "70000" is out of range
for type smallint`, at the literal (position 8), as PostgreSQL does; so does
an out-of-range number inside an `int2vector`, `oidvector` or array literal.
The error carried no position before.

#### Fixed

- An input function's 22003 for a constant is placed at the literal.
Fixes RP-866 (tests/backlog/test_rp_found_rp605.py::test_out_of_range_literal_has_a_position)
1 change: 1 addition & 0 deletions crates/secantus-pg/src/explain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2919,6 +2919,7 @@ mod heap_fetches_tests {
dummy_relation: false,
sort_keys: None,
series_scan: None,
series_qual: None,
}
}

Expand Down
14 changes: 14 additions & 0 deletions crates/secantus-pg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44829,6 +44829,20 @@ impl PgHandler {
}
}
}
// With a WHERE it prices the Filter's quals too (RP-847).
if tree.estimate.is_none()
&& tree.children.is_empty()
&& tree.name == "Function Scan on generate_series"
{
if let (Statement::Select(sel), Some(qual)) =
(inner.as_ref(), &options.series_qual)
{
if let Some(series) = &sel.series {
tree.estimate =
Some(scan_estimate::filtered_series_scan_estimate(series, qual));
}
}
}
// An INSERT, UPDATE or DELETE is a ModifyTable over a
// Result or the relation's sequential scan (RP-331).
if tree.children.len() == 1
Expand Down
19 changes: 19 additions & 0 deletions crates/secantus-pg/src/scan_estimate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4577,6 +4577,25 @@ pub(crate) fn series_scan_estimate(series: &secantus_pgplan::Series, columns: us
)
}

/// `cost_functionscan` of a `generate_series` scan with a Filter (RP-847):
/// every row the series yields pays `cpu_tuple_cost` and the quals, and
/// the estimate is the rows times their selectivity. The scan returns its
/// one column, which the Filter reads.
pub(crate) fn filtered_series_scan_estimate(
series: &secantus_pgplan::Series,
qual: &secantus_pgplan::SeriesQual,
) -> Estimate {
const CPU_OPERATOR_COST: f64 = 0.0025;
let (startup, _, rows, width) = series_scan_estimate(series, 1);
let per_row = costsize::CPU_TUPLE_COST + CPU_OPERATOR_COST * f64::from(qual.ops);
(
startup,
startup + per_row * rows,
clamp_rows(rows * qual.selectivity(rows)),
width,
)
}

#[cfg(test)]
mod series_tests {
use super::series_scan_estimate;
Expand Down
53 changes: 33 additions & 20 deletions crates/secantus-pgplan/src/errpos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,31 @@ fn element_literal(toks: &[Tok], value: &str) -> Option<usize> {
hits.next().is_none().then_some(i)
}

/// The token of the literal an input function read `value` from, or `None`
/// when no literal holds it or it reached the function as a text value.
fn input_literal(toks: &[Tok], value: &str) -> Option<usize> {
let lit = format!("'{}'", value.replace('\'', "''"));
let i = toks
.iter()
.position(|t| t.text == lit)
.or_else(|| element_literal(toks, value))?;
// A literal cast to a string type first is a text VALUE when the
// target's input function reads it, at execution, where there is
// no position (`int8('x'::text)`, measured on 15.19).
let string_type = |t: &str| {
["text", "varchar", "name", "bpchar"]
.iter()
.any(|k| t.eq_ignore_ascii_case(k))
};
if toks.get(i + 1).is_some_and(|t| t.text == "::")
&& toks.get(i + 2).is_some_and(|t| string_type(t.text))
&& !toks.get(i + 3).is_some_and(|t| t.text.starts_with('['))
{
return None;
}
Some(toks[i].start)
}

/// `ALTER TABLE | INDEX | VIEW | SEQUENCE | MATERIALIZED VIEW | FOREIGN
/// TABLE`, whose missing relation PostgreSQL reports without a position.
pub fn alter_finds_relation_by_name(sql: &str) -> bool {
Expand Down Expand Up @@ -713,26 +738,14 @@ pub fn error_position(sql: &str, sqlstate: &str, message: &str) -> Option<usize>
// `invalid input syntax for type integer: "abc"`: the literal.
let (_, value) = m.rsplit_once(": \"")?;
let value = value.strip_suffix('"')?;
let lit = format!("'{}'", value.replace('\'', "''"));
let i = toks
.iter()
.position(|t| t.text == lit)
.or_else(|| element_literal(&toks, value))?;
// A literal cast to a string type first is a text VALUE when the
// target's input function reads it, at execution, where there is
// no position (`int8('x'::text)`, measured on 15.19).
let string_type = |t: &str| {
["text", "varchar", "name", "bpchar"]
.iter()
.any(|k| t.eq_ignore_ascii_case(k))
};
if toks.get(i + 1).is_some_and(|t| t.text == "::")
&& toks.get(i + 2).is_some_and(|t| string_type(t.text))
&& !toks.get(i + 3).is_some_and(|t| t.text.starts_with('['))
{
return None;
}
pos(toks[i].start)
input_literal(&toks, value).and_then(pos)
}
// An input function's `value "70000" is out of range for type
// smallint`: at the literal, as its 22P02 is.
"22003" => {
let rest = m.strip_prefix("value \"")?;
let (value, _) = rest.rsplit_once("\" is out of range for type ")?;
input_literal(&toks, value).and_then(pos)
}
// `regoperator`'s own refusals of its input: at the literal.
"42P02" | "54023" if matches!(m, "missing argument" | "too many arguments") => {
Expand Down
195 changes: 190 additions & 5 deletions crates/secantus-pgplan/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3723,6 +3723,182 @@ pub struct ExplainOptions {
/// Scan names (when it differs from the function's), and its WHERE as
/// the scan's `Filter` (RP-847).
pub series_scan: Option<(Option<String>, Option<String>)>,
/// That WHERE priced as `cost_functionscan` prices the scan's quals
/// (RP-847): its operators per row, and its selectivity given the
/// series' row count.
pub series_qual: Option<SeriesQual>,
}

/// A `generate_series` scan's WHERE as the planner prices it (RP-847).
#[derive(Debug, Clone, PartialEq)]
pub struct SeriesQual {
/// `cost_qual_eval`'s operators per row, each one `cpu_operator_cost`.
pub ops: u32,
/// The conjuncts, each an OR of comparisons; see `selectivity`.
conjuncts: Vec<Vec<SeriesCmp>>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
enum SeriesCmp {
/// `=` (`eqsel`), or `<>` (`neqsel`) when negated.
Eq { negated: bool },
/// `<` `<=` (`below`) or `>` `>=` of the bare column against a
/// constant: a candidate for `addRangeClause`'s pairing.
ColumnIneq { below: bool },
/// Any other inequality: `DEFAULT_INEQ_SEL`.
Ineq,
}

impl SeriesQual {
/// `clauselist_selectivity` over a relation with no statistics of
/// `rows` tuples: `eqsel` is one over `get_variable_numdistinct`'s
/// default (the tuples, up to `DEFAULT_NUM_DISTINCT`), an inequality
/// `DEFAULT_INEQ_SEL`, a `<` / `>` pair on the column
/// `DEFAULT_RANGE_INEQ_SEL`, and an OR `s1 + s2 - s1 * s2`.
pub fn selectivity(&self, rows: f64) -> f64 {
const DEFAULT_INEQ_SEL: f64 = 0.3333333333333333;
const DEFAULT_RANGE_INEQ_SEL: f64 = 0.005;
let ndistinct = if rows < 200.0 {
rows.max(1.0).round()
} else {
200.0
};
let one = |c: &SeriesCmp| match c {
SeriesCmp::Eq { negated: false } => 1.0 / ndistinct,
SeriesCmp::Eq { negated: true } => 1.0 - 1.0 / ndistinct,
_ => DEFAULT_INEQ_SEL,
};
let (mut s, mut lo, mut hi) = (1.0, false, false);
for arms in &self.conjuncts {
match arms.as_slice() {
[SeriesCmp::ColumnIneq { below: true }] if !hi => hi = true,
[SeriesCmp::ColumnIneq { below: false }] if !lo => lo = true,
_ => {
let mut or = 0.0;
for a in arms {
let x = one(a);
or = or + x - or * x;
}
s *= or;
}
}
}
s * match (lo, hi) {
(true, true) => DEFAULT_RANGE_INEQ_SEL,
(true, false) | (false, true) => DEFAULT_INEQ_SEL,
_ => 1.0,
}
}
}

/// A series scan's WHERE priced (RP-847), over the one column `column`;
/// `None` for a clause not modelled: a function call, a cast, a sublink,
/// a comparison reading no column (a constant gate, not a Filter).
fn series_qual(w: &pg_query::protobuf::Node, column: &str) -> Option<SeriesQual> {
fn operand(n: &pg_query::protobuf::Node, column: &str) -> Option<(u32, bool, bool)> {
// (operators, reads the column, is the bare column)
match n.node.as_ref()? {
N::AConst(_) => Some((0, false, false)),
N::ColumnRef(c) => match c.fields.last()?.node.as_ref()? {
N::String(s) if s.sval == column => Some((0, true, true)),
_ => None,
},
N::AExpr(e) if e.kind == AExprKind::AexprOp as i32 => {
let op = match e.name.last()?.node.as_ref()? {
N::String(s) => s.sval.as_str(),
_ => return None,
};
if !["+", "-", "*", "/", "%"].contains(&op) {
return None;
}
let (mut ops, mut reads) = (0, false);
for side in [e.lexpr.as_deref(), e.rexpr.as_deref()]
.into_iter()
.flatten()
{
let (o, r, _) = operand(side, column)?;
ops += o;
reads |= r;
}
Some((if reads { ops + 1 } else { 0 }, reads, false))
}
_ => None,
}
}
fn cmp(n: &pg_query::protobuf::Node, column: &str) -> Option<(u32, SeriesCmp)> {
let N::AExpr(e) = n.node.as_ref()? else {
return None;
};
if e.kind != AExprKind::AexprOp as i32 {
return None;
}
let op = match e.name.last()?.node.as_ref()? {
N::String(s) => s.sval.clone(),
_ => return None,
};
let l = operand(e.lexpr.as_deref()?, column)?;
let r = operand(e.rexpr.as_deref()?, column)?;
if l.1 == r.1 {
return None;
}
// The column's side first, the operator commuted to match.
let (bare, op) = if l.1 {
(l.2, op)
} else {
let c = match op.as_str() {
"<" => ">",
">" => "<",
"<=" => ">=",
">=" => "<=",
o => o,
};
(r.2, c.to_string())
};
let kind = match op.as_str() {
"=" => SeriesCmp::Eq { negated: false },
"<>" | "!=" => SeriesCmp::Eq { negated: true },
"<" | "<=" if bare => SeriesCmp::ColumnIneq { below: true },
">" | ">=" if bare => SeriesCmp::ColumnIneq { below: false },
"<" | "<=" | ">" | ">=" => SeriesCmp::Ineq,
_ => return None,
};
Some((l.0 + r.0 + 1, kind))
}
fn arms(n: &pg_query::protobuf::Node, column: &str, out: &mut Vec<SeriesCmp>) -> Option<u32> {
if let Some(N::BoolExpr(b)) = n.node.as_ref() {
if BoolExprType::try_from(b.boolop) == Ok(BoolExprType::OrExpr) {
let mut ops = 0;
for a in &b.args {
ops += arms(a, column, out)?;
}
return Some(ops);
}
return None;
}
let (ops, c) = cmp(n, column)?;
out.push(c);
Some(ops)
}
let mut conj = Vec::new();
fn flatten<'a>(n: &'a pg_query::protobuf::Node, out: &mut Vec<&'a pg_query::protobuf::Node>) {
if let Some(N::BoolExpr(b)) = n.node.as_ref() {
if BoolExprType::try_from(b.boolop) == Ok(BoolExprType::AndExpr) {
for a in &b.args {
flatten(a, out);
}
return;
}
}
out.push(n);
}
flatten(w, &mut conj);
let (mut ops, mut conjuncts) = (0, Vec::new());
for c in conj {
let mut a = Vec::new();
ops += arms(c, column, &mut a)?;
conjuncts.push(a);
}
Some(SeriesQual { ops, conjuncts })
}

/// `CREATE [OR REPLACE] VIEW`.
Expand Down Expand Up @@ -11703,6 +11879,7 @@ fn plan_explain(
dummy_relation: false,
sort_keys: None,
series_scan: None,
series_qual: None,
};
let (mut timing, mut summary) = (None, None);
for o in &e.options {
Expand Down Expand Up @@ -11781,9 +11958,12 @@ fn plan_explain(
}
}
let scan_quals = explain_scan_quals(&query, lookup).unwrap_or_default();
options.series_scan = match &query {
N::SelectStmt(s) => explain_series_scan(s),
_ => None,
(options.series_scan, options.series_qual) = match &query {
N::SelectStmt(s) => match explain_series_scan(s) {
Some((scan, qual)) => (Some(scan), qual),
None => (None, None),
},
_ => (None, None),
};
options.sort_keys = match &query {
N::SelectStmt(s) => match s.from_clause.as_slice() {
Expand Down Expand Up @@ -52312,9 +52492,10 @@ fn scan_where(query: &N) -> Option<(String, &pg_query::protobuf::Node)> {
/// prints (RP-847): `ExplainNode` names the alias where it differs from
/// the function's name, and `show_scan_qual` prints the WHERE the scan
/// applies to each row, its conjuncts ANDed.
#[allow(clippy::type_complexity)]
fn explain_series_scan(
s: &pg_query::protobuf::SelectStmt,
) -> Option<(Option<String>, Option<String>)> {
) -> Option<((Option<String>, Option<String>), Option<SeriesQual>)> {
let [item] = s.from_clause.as_slice() else {
return None;
};
Expand Down Expand Up @@ -52368,7 +52549,11 @@ fn explain_series_scan(
})
}
};
Some((alias, filter))
let qual = s
.where_clause
.as_deref()
.and_then(|w| series_qual(w, &column));
Some(((alias, filter), qual))
}

/// A statement over ONE relation: its WHERE's conjuncts for EXPLAIN.
Expand Down
Loading
Loading