diff --git a/egglog-experimental/Cargo.lock b/egglog-experimental/Cargo.lock index 1ae1a16..ac8e301 100644 --- a/egglog-experimental/Cargo.lock +++ b/egglog-experimental/Cargo.lock @@ -330,7 +330,6 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "egglog" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "chrono", "clap", @@ -338,6 +337,7 @@ dependencies = [ "dyn-clone", "egglog-add-primitive", "egglog-ast", + "egglog-backend-trait", "egglog-bridge", "egglog-core-relations", "egglog-numeric-id", @@ -355,6 +355,7 @@ dependencies = [ "rayon", "rustc-hash", "serde_json", + "smallvec", "thiserror", "web-time", ] @@ -362,7 +363,6 @@ dependencies = [ [[package]] name = "egglog-add-primitive" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "quote", "syn 2.0.108", @@ -371,15 +371,24 @@ dependencies = [ [[package]] name = "egglog-ast" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "ordered-float", ] +[[package]] +name = "egglog-backend-trait" +version = "2.0.0" +dependencies = [ + "anyhow", + "egglog-bridge", + "egglog-core-relations", + "egglog-numeric-id", + "egglog-reports", +] + [[package]] name = "egglog-bridge" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "anyhow", "dyn-clone", @@ -402,7 +411,6 @@ dependencies = [ [[package]] name = "egglog-concurrency" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "arc-swap", "bumpalo", @@ -414,7 +422,6 @@ dependencies = [ [[package]] name = "egglog-core-relations" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "anyhow", "bumpalo", @@ -457,7 +464,6 @@ dependencies = [ [[package]] name = "egglog-numeric-id" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "rayon", ] @@ -465,7 +471,6 @@ dependencies = [ [[package]] name = "egglog-reports" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "clap", "hashbrown 0.16.0", @@ -479,7 +484,6 @@ dependencies = [ [[package]] name = "egglog-union-find" version = "2.0.0" -source = "git+https://github.com/egraphs-good/egglog.git?rev=5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd#5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd" dependencies = [ "crossbeam", "egglog-concurrency", diff --git a/egglog-experimental/Cargo.toml b/egglog-experimental/Cargo.toml index 52f260a..7b0044b 100644 --- a/egglog-experimental/Cargo.toml +++ b/egglog-experimental/Cargo.toml @@ -17,9 +17,11 @@ default = ["bin"] bin = ["egglog/bin"] [dependencies] -egglog = { git = "https://github.com/egraphs-good/egglog.git", rev = "5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd", default-features = false } -egglog-ast = { git = "https://github.com/egraphs-good/egglog.git", rev = "5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd", default-features = false } -egglog-reports = { git = "https://github.com/egraphs-good/egglog.git", rev = "5294cdc66a7b90a9a1480cb2d930f2ee5785c8dd", default-features = false } +# Point at the LOCAL egglog fork (../egglog) so experimental tracks this repo's +# egglog (including the Backend-trait work), per the monorepo's intent. +egglog = { path = "../egglog", default-features = false } +egglog-ast = { path = "../egglog/egglog-ast", default-features = false } +egglog-reports = { path = "../egglog/egglog-reports", default-features = false } num = "0.4.3" lazy_static = "1.4" diff --git a/egglog-experimental/flowlog/.gitignore b/egglog-experimental/flowlog/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/egglog-experimental/flowlog/.gitignore @@ -0,0 +1 @@ +/target diff --git a/egglog-experimental/flowlog/Cargo.lock b/egglog-experimental/flowlog/Cargo.lock new file mode 100644 index 0000000..8bc9992 --- /dev/null +++ b/egglog-experimental/flowlog/Cargo.lock @@ -0,0 +1,1556 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "columnar" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94063ab5cdb61ed8e5a8294c3c7cc8466c1a9eafc7b9ca02b2a521ea06e52d92" +dependencies = [ + "bytemuck", + "columnar_derive", + "smallvec", +] + +[[package]] +name = "columnar_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d11c5dbac40214de1027e4783272d192198c986d94b65ae43c0886b4fb58ec5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "columnation" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90b1614014f6958477dcdb77a2d489555db48ca61efe94c5cde2b0446933ed1" +dependencies = [ + "paste", + "smallvec", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "differential-dataflow" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "befdc46c04f5671a2dfe32a9d06ce8efe44f650f0f6f974d190b9a6ee1c17665" +dependencies = [ + "columnar", + "columnation", + "fnv", + "paste", + "serde", + "smallvec", + "timely", +] + +[[package]] +name = "differential-dogs3" +version = "0.24.0" +dependencies = [ + "differential-dataflow", + "serde", + "timely", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "egglog" +version = "2.0.0" +dependencies = [ + "csv", + "dyn-clone", + "egglog-add-primitive", + "egglog-ast", + "egglog-backend-trait", + "egglog-bridge", + "egglog-core-relations", + "egglog-numeric-id", + "egglog-reports", + "egraph-serialize", + "enum-map", + "hashbrown 0.16.1", + "im-rc", + "indexmap", + "log", + "num", + "ordered-float", + "rayon", + "rustc-hash", + "serde_json", + "smallvec", + "thiserror", + "web-time", +] + +[[package]] +name = "egglog-add-primitive" +version = "2.0.0" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "egglog-ast" +version = "2.0.0" +dependencies = [ + "ordered-float", +] + +[[package]] +name = "egglog-backend-trait" +version = "2.0.0" +dependencies = [ + "anyhow", + "egglog-bridge", + "egglog-core-relations", + "egglog-numeric-id", + "egglog-reports", +] + +[[package]] +name = "egglog-bridge" +version = "2.0.0" +dependencies = [ + "anyhow", + "dyn-clone", + "egglog-core-relations", + "egglog-numeric-id", + "egglog-reports", + "egglog-union-find", + "hashbrown 0.16.1", + "indexmap", + "log", + "num-rational", + "once_cell", + "ordered-float", + "rayon", + "smallvec", + "thiserror", + "web-time", +] + +[[package]] +name = "egglog-concurrency" +version = "2.0.0" +dependencies = [ + "arc-swap", + "bumpalo", + "egglog-numeric-id", + "rayon", + "smallvec", +] + +[[package]] +name = "egglog-core-relations" +version = "2.0.0" +dependencies = [ + "anyhow", + "bumpalo", + "crossbeam", + "crossbeam-queue", + "dashmap", + "dyn-clone", + "egglog-concurrency", + "egglog-numeric-id", + "egglog-reports", + "egglog-union-find", + "fixedbitset", + "hashbrown 0.16.1", + "indexmap", + "log", + "num", + "once_cell", + "rand 0.9.4", + "rayon", + "rustc-hash", + "smallvec", + "thiserror", + "web-time", +] + +[[package]] +name = "egglog-experimental-flowlog" +version = "0.1.0" +dependencies = [ + "anyhow", + "differential-dataflow", + "differential-dogs3", + "dyn-clone", + "egglog", + "egglog-backend-trait", + "egglog-core-relations", + "egglog-numeric-id", + "flowlog-build", + "flowlog-runtime", + "hashbrown 0.16.1", + "log", + "serde", + "timely", +] + +[[package]] +name = "egglog-numeric-id" +version = "2.0.0" +dependencies = [ + "rayon", +] + +[[package]] +name = "egglog-reports" +version = "2.0.0" +dependencies = [ + "clap", + "hashbrown 0.16.1", + "indexmap", + "rustc-hash", + "serde", + "serde_json", + "web-time", +] + +[[package]] +name = "egglog-union-find" +version = "2.0.0" +dependencies = [ + "crossbeam", + "egglog-concurrency", + "egglog-numeric-id", +] + +[[package]] +name = "egraph-serialize" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0977732fb537ace6f8c15ce160ebdda78b6502b4866d3b904e4fe752e2be4702" +dependencies = [ + "indexmap", + "once_cell", + "ordered-float", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flowlog-build" +version = "0.3.1" +source = "git+https://github.com/flowlog-rs/flowlog?rev=fb0eeb1e9989a24478459a2bfbe3192d8475eb7d#fb0eeb1e9989a24478459a2bfbe3192d8475eb7d" +dependencies = [ + "clap", + "codespan-reporting", + "itertools", + "ordered-float", + "pest", + "pest_derive", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "thiserror", + "tracing", +] + +[[package]] +name = "flowlog-runtime" +version = "0.2.3" +source = "git+https://github.com/flowlog-rs/flowlog?rev=fb0eeb1e9989a24478459a2bfbe3192d8475eb7d#fb0eeb1e9989a24478459a2bfbe3192d8475eb7d" +dependencies = [ + "differential-dataflow", + "lasso", + "ordered-float", + "regex", + "serde", + "timely", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", + "serde", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "im-rc" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", + "serde", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", + "rand 0.8.6", + "serde", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "rand_core 0.6.4", + "serde", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "serde", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "timely" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5af6859fde675a87f12da672e0bdf08bfc809e68d8c6728f50942d09c0480a3" +dependencies = [ + "bincode", + "byteorder", + "columnar", + "columnation", + "getopts", + "itertools", + "serde", + "smallvec", + "timely_bytes", + "timely_communication", + "timely_container", + "timely_logging", +] + +[[package]] +name = "timely_bytes" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b427b919d60d815c9c30b52d8080ebde599380db18c7c98a8c99fc172601ca1" + +[[package]] +name = "timely_communication" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2137e50f9c368c04d9971b7909c6fceadedb24c6e4fa4173bf55bf6d415329db" +dependencies = [ + "byteorder", + "columnar", + "getopts", + "serde", + "timely_bytes", + "timely_container", + "timely_logging", +] + +[[package]] +name = "timely_container" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45d41809553b99db9abc2ad2c5f5b8f755df1f2191051527deaadeea12d5a7e" + +[[package]] +name = "timely_logging" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00582f3cfc652e3173eb9a140981133f359029e6b746667414a8d334285163d6" +dependencies = [ + "timely_container", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/egglog-experimental/flowlog/Cargo.toml b/egglog-experimental/flowlog/Cargo.toml new file mode 100644 index 0000000..35d40ca --- /dev/null +++ b/egglog-experimental/flowlog/Cargo.toml @@ -0,0 +1,45 @@ +# A FlowLog (differential-dataflow) backend for egglog, living OUTSIDE the +# egglog crate — it depends only on the public `egglog-backend-trait` SPI plus +# the neutral `egglog-core-relations`. This is the worked example that a third +# party can implement their own egglog backend against the `Backend` trait. +# +# It is its own workspace (empty `[workspace]` table below) so its heavy, +# edition-2021 differential-dataflow / timely / flowlog-rs dependency tree does +# not leak into the edition-2024 egglog workspace. +[package] +name = "egglog-experimental-flowlog" +version = "0.1.0" +edition = "2021" +description = "A FlowLog (differential-dataflow)-backed egglog Backend, as an out-of-tree example of the egglog-backend-trait SPI." +license = "MIT" +publish = false + +[workspace] + +[dependencies] +anyhow = "1" +egglog-backend-trait = { path = "../../egglog/egglog-backend-trait" } +egglog-core-relations = { path = "../../egglog/core-relations" } +egglog-numeric-id = { path = "../../egglog/numeric-id" } +hashbrown = "0.16" +log = "0.4" +dyn-clone = "1.0.17" +# flowlog-runtime: the generated incremental engine's string interning / txn +# state / differential-dataflow re-exports. Pinned git rev (reproducible). +flowlog-runtime = { git = "https://github.com/flowlog-rs/flowlog", rev = "fb0eeb1e9989a24478459a2bfbe3192d8475eb7d" } +# Raw differential-dataflow + timely, used in-process by `dd_native.rs`. +differential-dataflow = "=0.24.0" +timely = "=0.30.0" +# `--wcoj` worst-case-optimal join operators: vendored + one-line-patched +# `differential-dogs3` (pins the SAME dd 0.24.0 / timely 0.30.0). +differential-dogs3 = { path = "vendor/differential-dogs3" } +serde = { version = "1", features = ["derive", "rc"] } + +[build-dependencies] +# Compiles the bundled `transitive_step.dl` into a Rust module at build time. +flowlog-build = { git = "https://github.com/flowlog-rs/flowlog", rev = "fb0eeb1e9989a24478459a2bfbe3192d8475eb7d" } + +[dev-dependencies] +# The frontend, so the integration test can run real `.egg` files on the +# FlowLog backend via `egglog::EGraph::with_backend(..)`. +egglog = { path = "../../egglog", default-features = false } diff --git a/egglog-experimental/flowlog/build.rs b/egglog-experimental/flowlog/build.rs new file mode 100644 index 0000000..2b8a933 --- /dev/null +++ b/egglog-experimental/flowlog/build.rs @@ -0,0 +1,33 @@ +//! Compile the build-time-fixed `transitive_step.dl` into a Rust module, in +//! INCREMENTAL mode so the generated engine is a long-lived +//! `DatalogIncrementalEngine` (insert_/remove_ staging + `commit()`), not a +//! one-shot batch `run()`. `lib.rs` include!s the generated module. +//! +//! API per `crates/flowlog-build/src/lib.rs` (confirmed by the spike): +//! Builder::default().mode(ExecutionMode::DatalogInc).compile(&["..dl"], &[]) +//! +//! `compile()` writes `$OUT_DIR/.rs` (stem = `.dl` file stem, here +//! `transitive_step.rs`). The generated file is self-contained: it +//! `pub use`s an inner module that itself pulls in the flowlog-runtime +//! re-exports of timely / differential-dataflow, so the include site needs no +//! extra imports. +//! +//! ## Milestone-1 scope note (the FlowLog crux) +//! +//! egglog defines rules at RUNTIME, but flowlog compiles `.dl` -> Rust at BUILD +//! time. For the M1 PROOF a fixed `.dl` is acceptable (per the brief). Runtime +//! rule installation (regenerate + rustc + dlopen, or another mechanism) is the +//! FlowLog analog of Feldera's static-circuit-rebuild risk and is investigated +//! in ../MILESTONE1.md; it is deferred to M2. +use flowlog_build::{Builder, ExecutionMode}; + +#[allow(clippy::disallowed_macros)] // for println! (cargo: directives) +fn main() { + println!("cargo:rerun-if-changed=transitive_step.dl"); + Builder::default() + // M1's transitive-closure proof is integer-only; no string interning. + .string_intern(false) + .mode(ExecutionMode::DatalogInc) + .compile(&["transitive_step.dl"], &[] as &[&str]) + .expect("flowlog-build failed to compile transitive_step.dl"); +} diff --git a/egglog-experimental/flowlog/src/codegen.rs b/egglog-experimental/flowlog/src/codegen.rs new file mode 100644 index 0000000..99c20a3 --- /dev/null +++ b/egglog-experimental/flowlog/src/codegen.rs @@ -0,0 +1,219 @@ +//! Runtime translation of an egglog rule (the trait-level `RuleBuilderOps` IR) +//! into a FlowLog `.dl` program plus a thin **driver** crate, for the M2 +//! shell-out architecture. +//! +//! ## What is generated, at runtime +//! +//! M1 compiled a *build-time-fixed* `transitive_step.dl` and recognized the +//! rule shape structurally. M2 instead takes the rule the frontend builds **at +//! runtime** (`StepShape`, recognized from the live `RuleIr`) and emits: +//! +//! 1. a `.dl` for it (a non-recursive single-join step — one `commit()` = one +//! bounded hop, exactly the M1 per-iteration model, but the relation names, +//! join columns, and head come from the runtime rule, not a fixed file); +//! 2. a driver `main.rs` that `include!`s the flowlog-build-generated engine and +//! speaks a line-based **stdin/stdout command protocol** (see `protocol`); +//! 3. a `Cargo.toml` + `build.rs` so the temp crate compiles standalone. +//! +//! The caller (`subprocess.rs`) hashes the `.dl`, builds the crate **once** per +//! rule-set (`cargo build`, ~tens of seconds cold), caches the binary by hash, +//! then spawns it and drives it over the pipe for the whole program — so the +//! flowlog incremental engine stays warm and incrementality is preserved. +//! +//! ## The non-recursive single-join shape +//! +//! The recognized step is `head(x, z) :- path(x, y), edge(y, z)`. We emit two +//! command-staged input relations (`path`, `edge`) and one output `hop`: +//! +//! ```text +//! .decl edge(src: int32, dst: int32) +//! .input edge(IO="command", delimiter=",") +//! .decl path(src: int32, dst: int32) +//! .input path(IO="command", delimiter=",") +//! .decl hop(src: int32, dst: int32) +//! hop(x, z) :- path(x, y), edge(y, z). +//! .output hop +//! ``` +//! +//! The host (`lib.rs::run_one_hop_shellout`) folds each commit's `hop` deltas +//! into the Rust-side mirror and re-stages the new `path` rows next round, the +//! same bounded host-feedback loop M1 used — only now driving a *subprocess* +//! over a pipe instead of an in-process engine. + +/// The fixed relation names the generated `.dl` and driver use. The runtime +/// rule's actual `FunctionId`s are mapped onto these three roles by the host +/// (`StepShape`); the `.dl` itself only needs stable lowercase idents so the +/// generated engine's API (`insert_path` / `insert_edge` / `IncrementalResults. +/// hop`) and the driver's protocol dispatch are predictable. +pub const REL_EDGE: &str = "edge"; +pub const REL_PATH: &str = "path"; +pub const REL_HOP: &str = "hop"; + +/// Emit the runtime `.dl` for a recognized transitive-closure step. +/// +/// The shape is fixed (non-recursive single join), but this is produced *at +/// rule-install time from the runtime rule IR* — it is not a checked-in file. +/// Future shapes (different arities / multiple joins) extend here. +pub fn emit_dl() -> String { + let mut s = String::new(); + s.push_str("// AUTO-GENERATED at runtime by egglog-bridge-flowlog (M2 shell-out).\n"); + s.push_str("// One non-recursive join = one bounded egglog hop per commit().\n\n"); + s.push_str(&format!(".decl {REL_EDGE}(src: int32, dst: int32)\n")); + s.push_str(&format!( + ".input {REL_EDGE}(IO=\"command\", delimiter=\",\")\n\n" + )); + s.push_str(&format!(".decl {REL_PATH}(src: int32, dst: int32)\n")); + s.push_str(&format!( + ".input {REL_PATH}(IO=\"command\", delimiter=\",\")\n\n" + )); + s.push_str(&format!(".decl {REL_HOP}(src: int32, dst: int32)\n")); + s.push_str(&format!( + "{REL_HOP}(x, z) :- {REL_PATH}(x, y), {REL_EDGE}(y, z).\n" + )); + s.push_str(&format!(".output {REL_HOP}\n")); + s +} + +/// The driver crate's `Cargo.toml`, parameterized by the absolute path to the +/// local flowlog-rs clone (so the temp crate finds `flowlog-runtime` / +/// `flowlog-build`). The crate is intentionally NOT a workspace member +/// (`[workspace]` empty table) so it builds standalone in its temp dir. +pub fn emit_cargo_toml(crate_name: &str, flowlog_root: &str) -> String { + format!( + r#"[package] +name = "{crate_name}" +version = "0.0.0" +edition = "2021" +publish = false + +# Standalone: do NOT inherit the egglog workspace. +[workspace] + +[[bin]] +name = "{crate_name}" +path = "src/main.rs" + +[dependencies] +flowlog-runtime = {{ path = "{flowlog_root}/crates/flowlog-runtime" }} + +[build-dependencies] +flowlog-build = {{ path = "{flowlog_root}/crates/flowlog-build" }} +"# + ) +} + +/// The driver crate's `build.rs`: compile the runtime-emitted `program.dl` into +/// `$OUT_DIR/program.rs` in incremental mode. Identical in spirit to the M1 +/// build.rs, but the `.dl` it compiles was written at runtime. +pub fn emit_build_rs() -> String { + r#"//! AUTO-GENERATED driver build.rs (M2 shell-out). +use flowlog_build::{Builder, ExecutionMode}; + +fn main() { + println!("cargo:rerun-if-changed=program.dl"); + Builder::default() + .string_intern(false) + .mode(ExecutionMode::DatalogInc) + .compile(&["program.dl"], &[] as &[&str]) + .expect("flowlog-build failed to compile program.dl"); +} +"# + .to_string() +} + +/// The driver `main.rs`: include the generated engine, then run the line-based +/// stdin/stdout command protocol. The host side that speaks this wire format is +/// [`crate::subprocess::DriverHandle`]. +/// +/// Protocol (one command per line on stdin; responses on stdout): +/// - `insert ` -> stage an insert delta (no response) +/// - `remove ` -> stage a remove delta (no response) +/// - `commit` -> step ONE epoch; emit `delta hop ` for every +/// output delta, then a final `ok` line +/// - `read ` -> (reserved; emits `ok`) +/// - `quit` -> exit 0 +/// +/// The driver flushes stdout after every command so the host never deadlocks. +pub fn emit_main_rs() -> String { + r#"//! AUTO-GENERATED driver (M2 shell-out): embeds a flowlog-rs +//! `DatalogIncrementalEngine` for the runtime-emitted `program.dl` and drives +//! it over a line-based stdin/stdout command protocol. One `commit` command = +//! one engine `commit()` = one bounded egglog hop. + +#[allow(clippy::all)] +#[allow(dead_code)] +#[allow(unused)] +mod gen { + include!(concat!(env!("OUT_DIR"), "/program.rs")); +} + +use gen::DatalogIncrementalEngine; +use std::io::{BufRead, Write}; + +fn main() { + let mut engine = DatalogIncrementalEngine::new(1); + + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + + for line in stdin.lock().lines() { + let line = match line { + Ok(l) => l, + Err(_) => break, + }; + let line = line.trim(); + if line.is_empty() { + continue; + } + let mut it = line.split_whitespace(); + let cmd = it.next().unwrap_or(""); + match cmd { + "insert" | "remove" => { + let rel = it.next().unwrap_or(""); + let cols: Vec = it.filter_map(|t| t.parse::().ok()).collect(); + if cols.len() != 2 { + let _ = writeln!(out, "err expected 2 columns, got {}", cols.len()); + let _ = out.flush(); + continue; + } + let tup = (cols[0], cols[1]); + let items = vec![tup]; + match (cmd, rel) { + ("insert", "edge") => engine.insert_edge(items), + ("insert", "path") => engine.insert_path(items), + ("remove", "edge") => engine.remove_edge(items), + ("remove", "path") => engine.remove_path(items), + _ => { + let _ = writeln!(out, "err unknown rel {rel}"); + let _ = out.flush(); + } + } + } + "commit" => { + let results = engine.commit(); + for (t, d) in results.hop.into_iter() { + let t: (i32, i32) = t; + let _ = writeln!(out, "delta hop {} {} {}", t.0, t.1, d); + } + let _ = writeln!(out, "ok"); + let _ = out.flush(); + } + "read" => { + // Reserved: the host keeps its own mirror; nothing to stream. + let _ = writeln!(out, "ok"); + let _ = out.flush(); + } + "quit" => { + break; + } + other => { + let _ = writeln!(out, "err unknown command {other}"); + let _ = out.flush(); + } + } + } +} +"# + .to_string() +} diff --git a/egglog-experimental/flowlog/src/compile.rs b/egglog-experimental/flowlog/src/compile.rs new file mode 100644 index 0000000..2879f5c --- /dev/null +++ b/egglog-experimental/flowlog/src/compile.rs @@ -0,0 +1,180 @@ +//! Rule IR and row representation for the FlowLog backend. +//! +//! ## The load-bearing design choice (per the milestone brief) +//! +//! egglog's `(run N)` applies a ruleset **N times with bounded extension per +//! round** — a transitive-closure rule extends **N hops, NOT to full closure**. +//! FlowLog's incremental engine (`commit()`) would, on a *recursive* `.dl`, +//! saturate to a fixed point inside one `commit()`; that is *wrong* for +//! egglog's bounded iteration (it is the same trap the Feldera Phase-0 spike's +//! recursive DD scope fell into). +//! +//! So the M1 backend compiles a **non-recursive** flowlog program +//! (`transitive_step.dl`): `hop(x,z) :- path(x,y), edge(y,z).` One `commit()` +//! performs exactly **one round** of the join over the staged delta. The host +//! (`lib.rs::run_rules`) drives `(run N)` by feeding the previous round's new +//! `path` rows and calling `commit()` once per round — N calls = N hops. +//! +//! ## Row representation +//! +//! Every relation's rows are stored in the Rust-side mirror as a variable-width +//! boxed slice of `u32` (egglog [`Value`] reps), exactly `arity` columns wide. +//! `lookup_id` / `for_each` / `table_size` read the mirror. + +use egglog_backend_trait::{ExternalFunctionId, FunctionId, QueryEntry, Value}; + +/// Upper bound on relation arity (sanity check; the mirror row is +/// variable-width, so this is generous). +pub const MAX_ARITY: usize = 64; + +/// Uniform mirror row type: a variable-width boxed slice of `u32` +/// (egglog `Value` reps), exactly `arity` columns wide. +pub type Row = Box<[u32]>; + +/// How a function resolves a functional-dependency conflict (two rows sharing +/// the same key columns with different output columns). Recognized from the +/// trait `MergeFn` (see `lib.rs::add_table`). Mirrors the Feldera backend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MergeMode { + /// Plain relation: the whole row is the key, no output column to resolve. + Relation, + /// `:merge old` / `AssertEq`: keep the existing value on conflict. + Old, + /// `:merge new`: keep the new value on conflict. + New, + /// `:merge (ordering-min old new)` / `UnionId`: keep the numerically + /// smallest value (the union-find leader). Load-bearing for M2 rebuild. + Min, +} + +/// Pack a slice of `Value`s into a [`Row`] (exactly `vals.len()` columns). +pub fn pack_row(vals: &[Value]) -> Row { + use egglog_numeric_id::NumericId; + assert!( + vals.len() <= MAX_ARITY, + "row arity {} exceeds {MAX_ARITY}", + vals.len() + ); + vals.iter().map(|v| v.rep()).collect() +} + +/// Read column `i` (0-based) out of a [`Row`]. +#[inline] +pub fn row_col(r: &Row, i: usize) -> u32 { + r[i] +} + +/// Unpack the first `arity` columns of a [`Row`] into a `Vec`. +pub fn unpack_row(r: &Row, arity: usize) -> Vec { + use egglog_numeric_id::NumericId; + (0..arity).map(|i| Value::new(row_col(r, i))).collect() +} + +/// One column reference in a rule body atom or head action: either a bound +/// variable (identified by its [`egglog_backend_trait::VariableId`] rep) or a +/// constant value. +#[derive(Clone, Debug)] +pub enum Slot { + Var(u32), + Const(u32), +} + +/// Resolve a [`Slot`] to a concrete `u32` value: a constant resolves to itself, +/// a variable resolves through `get` (the current binding env), or `None` if the +/// variable is unbound. +pub fn slot_lookup(s: &Slot, get: &dyn Fn(u32) -> Option) -> Option { + match s { + Slot::Var(v) => get(*v), + Slot::Const(c) => Some(*c), + } +} + +impl Slot { + pub fn from_entry(e: &QueryEntry) -> Self { + use egglog_numeric_id::NumericId; + match e { + QueryEntry::Var(v) => Slot::Var(v.id.rep()), + QueryEntry::Const { val, .. } => Slot::Const(val.rep()), + } + } +} + +/// A body atom: a function table reference with one [`Slot`] per column. +#[derive(Clone, Debug)] +pub struct BodyAtom { + pub func: FunctionId, + pub slots: Vec, +} + +impl BodyAtom { + pub fn from_entries(func: FunctionId, entries: &[QueryEntry]) -> Self { + BodyAtom { + func, + slots: entries.iter().map(Slot::from_entry).collect(), + } + } +} + +/// One operation in a rule **body**, in emission order. +#[derive(Clone, Debug)] +pub enum BodyOp { + /// Match a table/relation atom. + Atom(BodyAtom), + /// Evaluate a primitive `func(args..)` (M2+; recorded but not executed in + /// the M1 flowlog-driven path). + #[allow(dead_code)] + Prim { + id: ExternalFunctionId, + args: Vec, + ret: Slot, + }, +} + +/// One operation in a rule **head**, in emission order. +#[derive(Clone, Debug)] +pub enum HeadOp { + /// `(set func(key..) = val)` / relation `insert` — slots are the full row. + Set { func: FunctionId, slots: Vec }, + /// `(delete func(key..))` — retraction (M2+). + #[allow(dead_code)] + Remove { func: FunctionId, slots: Vec }, + /// `(subsume func(key..))` (M2+). + #[allow(dead_code)] + Subsume { func: FunctionId, slots: Vec }, + /// RHS function lookup binding `ret` (M2+). + #[allow(dead_code)] + Lookup { + func: FunctionId, + args: Vec, + ret: u32, + }, + /// RHS primitive call binding `ret` (M2+). + #[allow(dead_code)] + Call { + id: ExternalFunctionId, + args: Vec, + ret: u32, + }, + /// `(union l r)` (M2+). + #[allow(dead_code)] + Union { l: Slot, r: Slot }, + /// `(panic msg)`. + #[allow(dead_code)] + Panic(String), +} + +/// The compiled IR for one egglog rule. +/// +/// `body` is an ordered list of table-atom matches (and, in later milestones, +/// primitive evaluations); `head` is an ordered list of writes. In M1 the +/// FlowLog-driven `run_rules` path recognizes the canonical +/// transitive-closure-step shape (two table body atoms joined on a shared +/// variable, one `set` head) and executes it through the bundled flowlog +/// incremental engine's `commit()`. The full ordered IR is retained so M2 can +/// add the host fallback / richer rule shapes the way Feldera M3 did. +#[derive(Clone, Debug, Default)] +pub struct RuleIr { + pub name: String, + pub body: Vec, + pub head: Vec, +} diff --git a/egglog-experimental/flowlog/src/dd_native.rs b/egglog-experimental/flowlog/src/dd_native.rs new file mode 100644 index 0000000..b639b68 --- /dev/null +++ b/egglog-experimental/flowlog/src/dd_native.rs @@ -0,0 +1,2771 @@ +//! In-process, build-once, epoch-driven incremental body join on RAW +//! `differential-dataflow` + `timely` — the FlowLog analog of the Feldera +//! backend's `dbsp_join::PersistentJoin`. +//! +//! This is the ONLY join path for the FlowLog `Interpret` backend (driven by +//! [`crate::interpret::run_iteration`]); there is no host nested-loop fallback. +//! It panics (via the caller) on shapes it does not support — see `plan_join`. +//! +//! ## Two implementations: per-rule vs FUSED per-ruleset +//! +//! [`FusedDdJoin`] is the PERF-CRITICAL path the interpreter drives: ONE shared +//! timely `Worker` hosts ONE dataflow for a whole RULESET, every distinct body +//! relation is a single SHARED input `Collection`, and each rule is a join +//! sub-stream reading those shared collections (the DD analog of feldera's +//! `FusedJoin`). [`PersistentDdJoin`] is the original per-RULE worker; it is +//! retained for the bridge-level incrementality unit test and documents the base +//! architecture below. Both feed only signed DELTAS into never-cleared +//! InputSessions, so the arrangements persist across epochs = incremental. +//! +//! ## The base architecture (mirrors feldera/DBSP) +//! +//! For each atom-bearing rule we build ONE differential-dataflow dataflow, ONCE, +//! inside a single-threaded timely `Worker` we OWN (so we can `step` it across +//! host calls). Each body atom occurrence is sourced from a +//! `differential_dataflow::input::InputSession`; the rule's body join is a +//! left-deep chain of DD `.join`s, with `!=` guards and value-prims inlined in +//! `.flat_map`/`.filter`; the head binding rows flow out through `.inspect_batch` +//! into a shared `Rc>>` capture buffer. +//! +//! Each egglog iteration (= one epoch) the host feeds ONLY the per-relation +//! signed DELTA into the InputSessions (`+1` insert, `-1` retract), advances the +//! timely timestamp, `step_while`s the worker to that epoch's fixpoint, and +//! drains the capture buffer to get the OUTPUT binding deltas. The +//! InputSessions are NEVER cleared — the DD arrangements persist across epochs, +//! which is what makes the join genuinely incremental (epoch K does only +//! delta·integral work, not a full recompute) — the whole point of the design. +//! +//! ## Fixpoint structure +//! +//! We use EXTERNAL epoch-drive (the host loop advances epochs and feeds head +//! outputs back as the next epoch's inputs), NOT an in-dataflow `iterate()` +//! scope. This matches egglog's bounded `(run N)` fire->rebuild->repeat model and +//! sidesteps DD `iterate()`'s monotonicity constraints under retraction (a +//! rebuild RETRACTS non-canonical rows, which `iterate()` cannot express +//! cleanly). The dataflow itself is NON-recursive: one epoch = one bounded hop. + +use std::cell::RefCell; +use std::rc::Rc; + +use anyhow::Result; +use differential_dataflow::input::InputSession; +// `--wcoj` triangle worst-case-optimal join: dogsdogsdogs prefix-extension / +// AltNeu delta-query operators (vendored + patched `differential-dogs3`). +use differential_dataflow::VecCollection; +use differential_dogs3::altneu::AltNeu; +use differential_dogs3::{CollectionIndex, ProposeExtensionMethod}; +use egglog_backend_trait::FunctionId; +use hashbrown::{HashMap, HashSet}; +use timely::communication::allocator::thread::Thread; +use timely::communication::allocator::Allocator; +use timely::dataflow::operators::probe::Handle as ProbeHandle; +use timely::dataflow::operators::{Inspect, Probe}; +use timely::dataflow::Scope; +use timely::worker::Worker; +use timely::WorkerConfig; + +use crate::compile::{BodyOp, RuleIr, Slot}; + +/// Fixed binding-row width (DD `Data` needs a `Sized + Ord + Hash` type; an +/// array gives us that without dbsp's `declare_tuples!`). Mirrors feldera's +/// `JOIN_WIDTH = 32` but bumped to 48 to cover the widest rebuild rule the +/// flowlog test corpus generates: `luminal-llama`'s `@rebuild_rule34` uses 35 +/// distinct body vars (a wide-arity congruence-closure rebuild). 48 covers +/// every reachable program with headroom; a rule exceeding this is reported as +/// a row-width-cap wall (raise `W` to extend coverage — it is purely a fixed +/// array size, costing `W * 4` bytes per binding row). +pub const W: usize = 48; + +/// A fixed-width binding / relation row flowing through the DD dataflow: +/// `row[i]` is the value of canonical body variable `i` (0 if not yet bound). +/// +/// A NEWTYPE over `[u32; W]` (rather than the bare array) because timely's +/// `ExchangeData` bound — required by DD `.join`/`.distinct` — is +/// `Serialize + Deserialize`, and `serde` only derives those for arrays up to +/// length 32. The hand-written serde impl (serialize as a fixed-length seq of +/// `W` `u32`s) lifts that cap so `W` can exceed 32 (the corpus needs 35). All +/// other derives (`Ord`/`Hash`/`Clone`/`Copy`) are auto for any array size. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct Row([u32; W]); + +/// The join key carried between DD `.join` stages: the shared bound columns +/// packed into a fixed-width array (others 0). Same newtype as [`Row`]. +type Key = Row; + +impl std::ops::Index for Row { + type Output = u32; + #[inline] + fn index(&self, i: usize) -> &u32 { + &self.0[i] + } +} + +impl std::ops::IndexMut for Row { + #[inline] + fn index_mut(&mut self, i: usize) -> &mut u32 { + &mut self.0[i] + } +} + +impl serde::Serialize for Row { + fn serialize(&self, s: S) -> Result { + use serde::ser::SerializeTuple; + // Fixed-length tuple of W u32s — bincode-friendly, no length prefix + // needed (the deserializer knows W). Sidesteps serde's 32-array cap. + let mut t = s.serialize_tuple(W)?; + for v in &self.0 { + t.serialize_element(v)?; + } + t.end() + } +} + +impl<'de> serde::Deserialize<'de> for Row { + fn deserialize>(d: D) -> Result { + struct RowVisitor; + impl<'de> serde::de::Visitor<'de> for RowVisitor { + type Value = Row; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "a tuple of {W} u32s") + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut a = [0u32; W]; + for (i, slot) in a.iter_mut().enumerate() { + *slot = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?; + } + Ok(Row(a)) + } + } + d.deserialize_tuple(W, RowVisitor) + } +} + +fn empty_row() -> Row { + Row([0u32; W]) +} + +impl Default for Row { + #[inline] + fn default() -> Self { + empty_row() + } +} + +/// SPIKE evidence flag: `FLOWLOG_DD_NATIVE_TRACE=1` prints per-epoch input/output +/// delta sizes to stderr (proof of incrementality + retraction). Off by default. +fn trace_enabled() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("FLOWLOG_DD_NATIVE_TRACE").is_some()) +} + +// --------------------------------------------------------------------------- +// Step-0 profiling counters (gated FLOWLOG_DD_PROF). Confirm/refute the +// per-rule-worker duplication hypothesis BEFORE refactoring: how many timely +// `Worker`s get spun up, how many `InputSession`s total, and where wall-time +// goes (worker.step vs the host-side prim re-run). Read+printed by `dd_prof_dump`. +// --------------------------------------------------------------------------- +use std::sync::atomic::{AtomicU64, Ordering}; +/// Number of timely `Worker`s created (one per `PersistentDdJoin::build`). +pub(crate) static PROF_WORKERS: AtomicU64 = AtomicU64::new(0); +/// Total `InputSession`s created across all workers (sum of atom occurrences). +pub(crate) static PROF_INPUT_SESSIONS: AtomicU64 = AtomicU64::new(0); +/// Total time spent in `worker.step_while` (the DD epoch fixpoint loop). +pub(crate) static PROF_STEP_NS: AtomicU64 = AtomicU64::new(0); +/// Total time spent feeding deltas into InputSessions + advancing/flushing. +pub(crate) static PROF_FEED_NS: AtomicU64 = AtomicU64::new(0); +/// Number of `step` calls that actually clocked the worker (pushed a delta). +pub(crate) static PROF_STEP_CALLS: AtomicU64 = AtomicU64::new(0); +/// Total time spent re-running body primitives host-side over the bindings. +pub(crate) static PROF_PRIM_NS: AtomicU64 = AtomicU64::new(0); +/// Total time spent computing the per-rule signed body-relation delta. +pub(crate) static PROF_DELTA_NS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn prof_enabled() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + // The low-level feed/step/call counters update whenever EITHER the global + // profile or the per-ruleset profile is requested — the per-ruleset path + // reads their before/after deltas around each `step` to attribute + // worker_step / feed time to the ruleset being run. + *ON.get_or_init(|| { + std::env::var_os("FLOWLOG_DD_PROF").is_some() + || std::env::var_os("FLOWLOG_DD_RULESET_PROF").is_some() + }) +} + +/// Per-ruleset profiling (gated `FLOWLOG_DD_RULESET_PROF`): attribute DD wall +/// time to the NAME of the ruleset being run, split into the same buckets as +/// the global profile, plus a call count and the summed input-delta row count. +pub(crate) fn ruleset_prof_enabled() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("FLOWLOG_DD_RULESET_PROF").is_some()) +} + +#[inline] +fn add_ns(c: &AtomicU64, d: std::time::Duration) { + c.fetch_add(d.as_nanos() as u64, Ordering::Relaxed); +} + +/// One ruleset's accumulated DD profile (nanoseconds + counts). +#[derive(Default, Clone)] +pub(crate) struct RulesetProf { + pub calls: u64, + pub total_ns: u64, + pub worker_step_ns: u64, + pub feed_ns: u64, + pub host_prim_ns: u64, + pub delta_compute_ns: u64, + pub delta_rows: u64, +} + +/// Accumulator keyed by ruleset NAME. Only touched when +/// `FLOWLOG_DD_RULESET_PROF` is set (zero overhead otherwise). +pub(crate) fn ruleset_prof_table() -> &'static std::sync::Mutex> { + use std::sync::OnceLock; + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| std::sync::Mutex::new(HashMap::new())) +} + +/// Record one ruleset's DD work for a single `run_rules` call. No-op unless +/// `FLOWLOG_DD_RULESET_PROF` is set. +#[allow(clippy::too_many_arguments)] +pub(crate) fn ruleset_prof_record( + ruleset: &str, + total_ns: u64, + worker_step_ns: u64, + feed_ns: u64, + host_prim_ns: u64, + delta_compute_ns: u64, + delta_rows: u64, +) { + if !ruleset_prof_enabled() { + return; + } + let mut table = ruleset_prof_table().lock().expect("ruleset prof lock"); + let e = table.entry(ruleset.to_string()).or_default(); + e.calls += 1; + e.total_ns += total_ns; + e.worker_step_ns += worker_step_ns; + e.feed_ns += feed_ns; + e.host_prim_ns += host_prim_ns; + e.delta_compute_ns += delta_compute_ns; + e.delta_rows += delta_rows; +} + +/// Print the Step-0 profile to stderr if `FLOWLOG_DD_PROF` is set. +pub fn dd_prof_dump() { + if !prof_enabled() { + return; + } + let workers = PROF_WORKERS.load(Ordering::Relaxed); + let sessions = PROF_INPUT_SESSIONS.load(Ordering::Relaxed); + let step_ns = PROF_STEP_NS.load(Ordering::Relaxed); + let feed_ns = PROF_FEED_NS.load(Ordering::Relaxed); + let calls = PROF_STEP_CALLS.load(Ordering::Relaxed); + let prim_ns = PROF_PRIM_NS.load(Ordering::Relaxed); + let delta_ns = PROF_DELTA_NS.load(Ordering::Relaxed); + #[allow(clippy::disallowed_macros)] + { + eprintln!( + "[FLOWLOG_DD_PROF] workers={workers} input_sessions={sessions} \ + nonempty_step_calls={calls} worker_step={:.3}s feed={:.3}s \ + host_prim={:.3}s delta_compute={:.3}s", + step_ns as f64 / 1e9, + feed_ns as f64 / 1e9, + prim_ns as f64 / 1e9, + delta_ns as f64 / 1e9, + ); + } +} + +/// Print the per-ruleset DD profile to stderr if `FLOWLOG_DD_RULESET_PROF` is +/// set: one row per ruleset, sorted by total DD time descending, with each +/// ruleset's share of the grand total. +pub fn dd_ruleset_prof_dump() { + if !ruleset_prof_enabled() { + return; + } + let table = ruleset_prof_table().lock().expect("ruleset prof lock"); + if table.is_empty() { + return; + } + let mut rows: Vec<(String, RulesetProf)> = + table.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + rows.sort_by(|a, b| b.1.total_ns.cmp(&a.1.total_ns)); + let grand_total: u64 = rows.iter().map(|(_, p)| p.total_ns).sum(); + let s = |ns: u64| ns as f64 / 1e9; + #[allow(clippy::disallowed_macros)] + { + eprintln!( + "[FLOWLOG_DD_RULESET_PROF] grand_total_dd={:.3}s", + s(grand_total) + ); + eprintln!( + "{:<28} {:>6} {:>9} {:>6} {:>11} {:>9} {:>9} {:>11}", + "ruleset", "calls", "total", "%dd", "worker_step", "feed", "host_prim", "delta_rows" + ); + for (name, p) in &rows { + let pct = if grand_total > 0 { + 100.0 * p.total_ns as f64 / grand_total as f64 + } else { + 0.0 + }; + eprintln!( + "{:<28} {:>6} {:>8.3}s {:>5.1}% {:>10.3}s {:>8.3}s {:>8.3}s {:>11}", + name, + p.calls, + s(p.total_ns), + pct, + s(p.worker_step_ns), + s(p.feed_ns), + s(p.host_prim_ns), + p.delta_rows, + ); + } + // Cross-check: share of fused worker_step in rules whose body reads @uf. + let uf_step = UF_BODY_STEP_NS.load(Ordering::Relaxed); + let all_step = ALL_STEP_NS.load(Ordering::Relaxed); + let uf_pct = if all_step > 0 { + 100.0 * uf_step as f64 / all_step as f64 + } else { + 0.0 + }; + eprintln!( + "[FLOWLOG_DD_RULESET_PROF] uf_body_worker_step={:.3}s of {:.3}s total worker_step \ + = {:.1}% (rules whose body reads a UF_* table)", + s(uf_step), + s(all_step), + uf_pct, + ); + } +} + +/// Cross-check accumulator for the per-ruleset profiler: nanoseconds of +/// (apportioned) fused worker_step attributable to rules whose BODY reads a +/// `@uf` table (`UF_*`), and the grand total worker_step nanos. Only touched +/// when `FLOWLOG_DD_RULESET_PROF` is set. +static UF_BODY_STEP_NS: AtomicU64 = AtomicU64::new(0); +static ALL_STEP_NS: AtomicU64 = AtomicU64::new(0); + +/// Record one `run_rules` call's worker_step split into UF-body-reading vs all. +pub(crate) fn ruleset_uf_body_record(uf_body_step_ns: u64, all_step_ns: u64) { + if !ruleset_prof_enabled() { + return; + } + UF_BODY_STEP_NS.fetch_add(uf_body_step_ns, Ordering::Relaxed); + ALL_STEP_NS.fetch_add(all_step_ns, Ordering::Relaxed); +} + +/// A planned DD join: canonical body-variable order + the table atoms. +pub struct JoinPlan { + /// `var_order[i]` is the variable id at binding-row column `i`. + var_order: Vec, + /// var id -> binding-row column index. + var_col: HashMap, + /// Body table atoms in emission order. + atoms: Vec, + /// `Some` ⇒ row projection (`FLOWLOG_PROJECT`) found a column-reusing layout + /// that fits `W` though the static var count does not; the binary chain uses + /// the per-step columns instead of `var_col`. `None` ⇒ the historical static + /// layout (every distinct var gets a permanent column). + projection: Option, +} + +struct PlanAtom { + func: FunctionId, + slots: Vec, +} + +impl JoinPlan { + /// The variable id at each CAPTURED binding-row column, in column order. With + /// projection this is the reduced surviving-var set (the build packs them into + /// columns `0..head_vars.len()`); without, it is the full static var order. + /// Either way the head scatter reads captured-row column `i` for var + /// `var_order()[i]`, so it stays correct. + pub fn var_order(&self) -> Vec { + match &self.projection { + Some(p) => p.head_vars.clone(), + None => self.var_order.clone(), + } + } +} + +/// Build the join plan for `rule`, or `Err(reason)` if the DD dataflow cannot +/// support its shape (the caller PANICS — there is no host fallback). Supported: +/// one or more table atoms, at most [`W`] distinct body vars, atom arity at most +/// [`W`]. Body prims (`!=` guards, value prims like `+`) are re-run host-side +/// over the bindings by the caller (the table-join-on-engine / +/// prim-tail-host-side split), so we accept them by leaving them to the host tail. +pub fn plan_join(rule: &RuleIr) -> Result { + let mut var_order: Vec = Vec::new(); + let mut var_col: HashMap = HashMap::new(); + let mut atoms: Vec = Vec::new(); + + let see = |v: u32, var_order: &mut Vec, var_col: &mut HashMap| { + if !var_col.contains_key(&v) { + var_col.insert(v, var_order.len()); + var_order.push(v); + } + }; + + for op in &rule.body { + match op { + BodyOp::Atom(atom) => { + if atom.slots.len() > W { + return Err(format!("atom arity {} > W {}", atom.slots.len(), W)); + } + for s in &atom.slots { + if let Slot::Var(v) = s { + see(*v, &mut var_order, &mut var_col); + } + } + atoms.push(PlanAtom { + func: atom.func, + slots: atom.slots.clone(), + }); + } + // Body prims (e.g. `!=` guards, value prims like `+`) are re-run + // host-side over the join bindings by the caller (the table-join-on- + // engine / prim-tail-host-side split). They do not affect join + // planning; a value prim may bind a fresh var the head reads. + BodyOp::Prim { .. } => {} + } + } + + if atoms.is_empty() { + return Err("no body table atoms (atom-less rule)".to_string()); + } + // ROW PROJECTION (gated `FLOWLOG_PROJECT`, default-off): when the static + // layout exceeds `W`, attempt a per-step register allocation that reuses + // binding-row columns for variables whose liveness intervals do not overlap + // (a body subterm var dies once the atom that consumes it has joined). If the + // reused-column frontier still fits `W`, the binary chain can run the rule on + // the existing fixed-width `Row` — this is what lets the giant Herbie seed + // rules (962 vars, frontier ~22) lower. Off ⇒ the historical bail below. + if project_enabled() { + if let Some(proj) = build_projection(&atoms, &rule.head, &rule.body) { + return Ok(JoinPlan { + var_order, + var_col, + atoms, + projection: Some(proj), + }); + } + // Projection could not fit `W` (frontier too wide) ⇒ fall through to the + // historical reject. + } + + if var_order.len() > W { + return Err(format!("too many body vars {} > W {}", var_order.len(), W)); + } + + Ok(JoinPlan { + var_order, + var_col, + atoms, + projection: None, + }) +} + +/// Is row projection enabled? Default-ON; set `FLOWLOG_NO_PROJECT` to fall back +/// to the static-column path. Projection is bit-exact (verified 49/49 vs the +/// bridge reference) and perf-neutral (it rides the existing merge map — no added +/// DD operator), and it is required for wide rules (e.g. Herbie's giant +/// ground-term seed, 962 vars) to lower at all, so it is on by default. +pub(crate) fn project_enabled() -> bool { + std::env::var_os("FLOWLOG_NO_PROJECT").is_none() +} + +/// A per-step binding-row column layout produced by [`build_projection`]: column +/// reuse (linear-scan register allocation over body-atom liveness) that keeps the +/// frontier within [`W`] so a rule whose STATIC var count exceeds `W` can still +/// run on the fixed-width `Row`. See the call site in [`plan_join`]. +#[derive(Clone, Debug)] +pub(crate) struct ProjectionPlan { + /// `step_col[i]` maps each variable LIVE during step `i` (atom `i`'s join) to + /// its binding-row column at that step. A var's column is stable from its + /// birth step to its death step but may differ across non-overlapping vars. + pub step_col: Vec>, + /// The surviving (head/body-prim-relevant) variables and their FINAL columns, + /// in a deterministic order. Drives the reduced head-scatter `var_order`. + pub head_vars: Vec, + /// Final column of each surviving var (parallel to `head_vars`). + pub head_cols: Vec, +} + +/// Build a column-reusing layout for the body atoms, or `None` if the reused +/// frontier still exceeds `W`. Liveness is first-use..last-use over the EMITTED +/// atom order, EXTENDED so any variable read by the head or a body prim stays +/// live to the end (it must survive into the captured row). Linear-scan slot +/// assignment: a column freed by a dead var is reused by a later var's birth. +fn build_projection( + atoms: &[PlanAtom], + head: &[crate::compile::HeadOp], + body: &[BodyOp], +) -> Option { + use hashbrown::HashSet; + let n = atoms.len(); + + // Variables the head / body prims read: these must survive to the end. + let mut survivor: HashSet = HashSet::new(); + collect_head_vars(head, &mut survivor); + for op in body { + if let BodyOp::Prim { args, ret, .. } = op { + for s in args { + if let Slot::Var(v) = s { + survivor.insert(*v); + } + } + if let Slot::Var(v) = ret { + survivor.insert(*v); + } + } + } + + // first[v]/last[v] = first/last atom index where v appears. Survivors get + // last = n-1 (live to the final step) so they are never freed mid-chain. + let mut first: HashMap = HashMap::new(); + let mut last: HashMap = HashMap::new(); + for (i, a) in atoms.iter().enumerate() { + for s in &a.slots { + if let Slot::Var(v) = s { + first.entry(*v).or_insert(i); + last.insert(*v, i); + } + } + } + for v in &survivor { + if let Some(l) = last.get_mut(v) { + *l = n - 1; + } + } + + // Vars born / dying at each step (in deterministic var-id order so the layout + // is reproducible). + let mut births: Vec> = vec![Vec::new(); n]; + let mut deaths: Vec> = vec![Vec::new(); n]; + let mut vars: Vec = first.keys().copied().collect(); + vars.sort_unstable(); + for v in vars { + births[first[&v]].push(v); + deaths[last[&v]].push(v); + } + + // Linear-scan register allocation. `col_of` is the current column of each + // live var; `free` is a stack of reusable columns (lowest first, so the + // layout is deterministic); `next` is the high-water column allocator. + let mut col_of: HashMap = HashMap::new(); + let mut free: Vec = Vec::new(); + let mut next: usize = 0; + let mut step_col: Vec> = Vec::with_capacity(n); + let mut peak = 0usize; + for i in 0..n { + // Births first (so this atom's fresh vars get columns before we snapshot). + for &v in &births[i] { + let c = if let Some(c) = free.pop() { + c + } else { + let c = next; + next += 1; + c + }; + col_of.insert(v, c); + } + peak = peak.max(col_of.len()); + if peak > W { + return None; + } + // Snapshot the layout AT this step (after births, before deaths) — every + // var the atom touches plus all still-live carried vars are present. + step_col.push(col_of.clone()); + // Deaths free their columns for reuse by later births. + for &v in &deaths[i] { + if let Some(c) = col_of.remove(&v) { + free.push(c); + } + // freeing low-first keeps assignment deterministic + free.sort_unstable_by(|a, b| b.cmp(a)); + } + } + + // Surviving vars + their FINAL columns (the columns they hold at the last + // step, where they are guaranteed live). Deterministic order by var id. + let final_layout = &step_col[n - 1]; + let mut head_vars: Vec = survivor + .into_iter() + .filter(|v| final_layout.contains_key(v)) + .collect(); + head_vars.sort_unstable(); + let head_cols: Vec = head_vars.iter().map(|v| final_layout[v]).collect(); + + Some(ProjectionPlan { + step_col, + head_vars, + head_cols, + }) +} + +/// Collect every variable a head op references into `out`. +fn collect_head_vars(head: &[crate::compile::HeadOp], out: &mut hashbrown::HashSet) { + use crate::compile::HeadOp; + let add = |s: &Slot, out: &mut hashbrown::HashSet| { + if let Slot::Var(v) = s { + out.insert(*v); + } + }; + for op in head { + match op { + HeadOp::Set { slots, .. } + | HeadOp::Remove { slots, .. } + | HeadOp::Subsume { slots, .. } => { + for s in slots { + add(s, out); + } + } + HeadOp::Lookup { args, ret, .. } => { + for s in args { + add(s, out); + } + out.insert(*ret); + } + HeadOp::Call { args, ret, .. } => { + for s in args { + add(s, out); + } + out.insert(*ret); + } + HeadOp::Union { l, r } => { + add(l, out); + add(r, out); + } + HeadOp::Panic(_) => {} + } + } +} + +// =========================================================================== +// `--wcoj` triangle worst-case-optimal join +// =========================================================================== +// +// Detects the reverse-distributivity triangle rule +// (rewrite (Add (Mul a b) (Mul a c)) (Mul a (Add b c))) +// which lowers (term encoding) to three arity-4 atoms +// A0 Mul(a, b, m1, x1) [a, b, m1, x1] +// A1 Mul(a, c, m2, x2) [a, c, m2, x2] +// A2 Add(m1, m2, o, x3) [m1, m2, o, x3] +// where A0,A1 are the SAME relation (a self-join sharing col-0 `a`), A2 is a +// different relation joining A0.col2=m1 and A1.col2=m2, and every named +// variable is distinct. The cyclic core is the triangle on (a, m1, m2): the +// binary join Mul_a ⋈ Mul_b on `a` materializes a `Σ_a deg(a)²` intermediate +// the WCOJ collapses to the output. + +/// The recognized triangle, with the BINDING-ROW column index of each of the 9 +/// distinct variables (a,b,m1,x1,c,m2,x2,o,x3) plus the two relation ids. All +/// downstream `extend_using` key selectors and the final `Row` assembly index +/// the binding row by these columns, so the WCOJ emits the SAME 9-column +/// binding row as the binary join (bit-exact head firing). +#[derive(Clone, Debug)] +pub(crate) struct TriangleShape { + mul_func: FunctionId, + add_func: FunctionId, + // binding-row columns (positions in the n_vars-wide Row) + col_a: usize, + col_b: usize, + col_m1: usize, + col_x1: usize, + col_c: usize, + col_m2: usize, + col_x2: usize, + col_o: usize, + col_x3: usize, +} + +/// Recognize the triangle shape in `plan`, or `None` if it does not match. +/// Structural (no name/func-id hardcoding): three arity-4 atoms, atoms 0 and 1 +/// the same relation sharing exactly their col-0 var, atom 2 a different +/// relation whose col-0 = atom0.col2 and col-1 = atom1.col2, and all 9 named +/// variables distinct. Stage 1 recognizes exactly this 3-atom triangle; +/// generalization is Stage 2. +pub(crate) fn detect_triangle(plan: &JoinPlan) -> Option { + if plan.atoms.len() != 3 { + return None; + } + // Each atom must be arity 4 with all-distinct Var slots (no consts, no + // repeated vars within an atom). + let atom_var = |i: usize| -> Option<[u32; 4]> { + let s = &plan.atoms[i].slots; + if s.len() != 4 { + return None; + } + let mut out = [0u32; 4]; + for (j, slot) in s.iter().enumerate() { + match slot { + Slot::Var(v) => out[j] = *v, + Slot::Const(_) => return None, + } + } + // distinct within the atom + for j in 0..4 { + for k in (j + 1)..4 { + if out[j] == out[k] { + return None; + } + } + } + Some(out) + }; + let a0 = atom_var(0)?; + let a1 = atom_var(1)?; + let a2 = atom_var(2)?; + + let mul_func = plan.atoms[0].func; + // A0, A1 same relation; A2 a different relation. + if plan.atoms[1].func != mul_func { + return None; + } + let add_func = plan.atoms[2].func; + if add_func == mul_func { + return None; + } + + // A0 = [a, b, m1, x1]; A1 = [a, c, m2, x2] — share col-0 (a), nothing else. + let (a, b, m1, x1) = (a0[0], a0[1], a0[2], a0[3]); + let (a_, c, m2, x2) = (a1[0], a1[1], a1[2], a1[3]); + if a != a_ { + return None; + } + // A2 = [m1, m2, o, x3] — col0 = A0.col2 (m1), col1 = A1.col2 (m2). + let (m1_, m2_, o, x3) = (a2[0], a2[1], a2[2], a2[3]); + if m1_ != m1 || m2_ != m2 { + return None; + } + + // All 9 named variables distinct (rules out degenerate self-overlap the + // generic shape would not produce; the binary join would treat such a rule + // differently, so we leave it to the binary path). + let vars = [a, b, m1, x1, c, m2, x2, o, x3]; + for i in 0..vars.len() { + for j in (i + 1)..vars.len() { + if vars[i] == vars[j] { + return None; + } + } + } + + let col = |v: u32| -> usize { plan.var_col[&v] }; + Some(TriangleShape { + mul_func, + add_func, + col_a: col(a), + col_b: col(b), + col_m1: col(m1), + col_x1: col(x1), + col_c: col(c), + col_m2: col(m2), + col_x2: col(x2), + col_o: col(o), + col_x3: col(x3), + }) +} + +// =========================================================================== +// Stage 2: GENERAL incremental WCOJ for an arbitrary >=3-atom CYCLIC body +// =========================================================================== +// +// Generalizes the hardcoded triangle (above) to any >=3-atom rule whose body +// join is CYCLIC — the class where a binary-join chain materializes an +// intermediate exceeding the output (the AGM blowup WCOJ removes). Acyclic and +// <=2-atom rules keep the binary `.join` chain (hybrid, Free-Join style): WCOJ +// buys nothing there and adds per-prefix index/intersection overhead. +// +// ## Detector (`detect_cyclic_cq`) +// +// Structural, no name/func-id dependence: +// 1. >= 3 table atoms, all slots distinct `Var`s within each atom, no consts +// (a const/repeated-var atom is a SELECTION the binary path handles; we +// leave it on the binary chain to stay conservative + bit-exact). +// 2. The body hypergraph is CYCLIC by GYO reduction (repeatedly drop an "ear" +// vertex appearing in <=1 remaining edge, and any edge subset of another; +// acyclic iff it reduces to empty). Cyclic ⇒ the binary chain blows up. +// 3. A full WCOJ plan is constructible (see `build_cq_plan`): for EVERY +// driver atom there is a variable order extending the driver one variable +// at a time, each step binding a variable that is the LAST unbound variable +// of some atom (so that atom keys entirely on already-bound columns). If +// any driver cannot be covered this way, we BAIL (return None → binary +// chain). This is the "only fire where provably bit-exact" stance: the +// construction mirrors the Stage-1 triangle exactly (full k-stream delta +// decomposition, ALL reads inside the AltNeu scope), and we only emit it +// when every variable is bound by an in-scope extend/propose. +// +// ## Plan (`CqPlan`) +// +// `var_col` (carried) + a `CqDriver` per atom. Each driver records its initial +// bound columns (the driver atom's vars, written straight from the delta row) +// then a sequence of `CqStep`s, each extending the prefix with ONE variable +// `bind_col` via a set of `CqExtender`s (`atom_idx` + the prefix columns the +// atom keys on + the relation column it proposes + whether the atom reads ALT +// (atom index < driver) or NEU (atom index > driver)). >=2 extenders for a step +// ⇒ multiway `extend` (the WCOJ intersection); exactly 1 ⇒ `propose_using`. + +/// One extender in a WCOJ extension step: read relation `atom_idx`'s `propose_col` +/// keyed by the prefix's `key_cols`, from the ALT (old) or NEU (new) trace. +#[derive(Clone, Debug)] +pub(crate) struct CqExtender { + /// Index into `CqPlan.atom_funcs` / the rule's atom list. + atom_idx: usize, + /// Binding-row columns (already bound in the prefix) this atom keys on. + /// Parallel to `key_atom_cols`. + key_cols: Vec, + /// The atom's relation columns (0-based slot positions) holding the key + /// vars, parallel to `key_cols` — used to build the atom's lookup index so + /// its key matches the prefix key_selector's columns. + key_atom_cols: Vec, + /// The relation column (0-based in the atom's row) that proposes the var. + propose_col: usize, + /// true ⇒ read the ALT (old) trace; false ⇒ the NEU (new) trace. + alt: bool, +} + +/// One JOIN-variable extension step: bind binding-row column `bind_col` from +/// `exts`. The intersected variable is a JOIN variable (appears in >=2 atoms). +/// With >=2 extenders this is the multiway `extend` (the WCOJ intersection that +/// collapses the intermediate); with a single extender it is a `propose_using`. +#[derive(Clone, Debug)] +pub(crate) struct CqStep { + /// The binding-row column this step writes. + bind_col: usize, + exts: Vec, +} + +/// A per-atom PAYLOAD recovery: once an atom's JOIN columns are all bound, +/// recover its payload variables (those appearing only in this atom) in ONE +/// `propose_using` keyed on the atom's join columns. `payload_cols[k]` = +/// `(binding_col, relation_col)` for the k-th payload var. +#[derive(Clone, Debug)] +pub(crate) struct CqRecover { + atom_idx: usize, + /// Binding columns (already bound) the atom keys on. Parallel to + /// `key_atom_cols`. + key_cols: Vec, + /// The atom's relation columns holding the key (join) vars. + key_atom_cols: Vec, + /// `(binding_col, relation_col)` per payload var to write from the proposal. + payload: Vec<(usize, usize)>, + /// true ⇒ read the ALT (old) trace; false ⇒ the NEU (new) trace. + alt: bool, +} + +/// One delta query `dQ/dA_driver`: seed the prefix from the driver atom's delta +/// row, bind every JOIN variable via `steps` (the WCOJ core), then recover each +/// non-driver atom's payload via `recovers`. ALL inside the AltNeu scope. +#[derive(Clone, Debug)] +pub(crate) struct CqDriver { + /// The driver atom's index. + driver_idx: usize, + /// `(binding_col, relation_col)` for each of the driver atom's variables — + /// written straight from the delta row into the initial prefix. + seed: Vec<(usize, usize)>, + steps: Vec, + recovers: Vec, +} + +/// A general WCOJ plan for a cyclic >=3-atom body. +#[derive(Clone, Debug)] +pub(crate) struct CqPlan { + /// Relation id per atom (atom order = the rule's `plan.atoms` order). + atom_funcs: Vec, + /// One delta query per atom. + drivers: Vec, +} + +/// The variables of an atom (its distinct `Var` slots), or `None` if the atom +/// has any const or repeated var (we leave those selections on the binary path). +fn atom_distinct_vars(slots: &[Slot]) -> Option> { + let mut out: Vec = Vec::with_capacity(slots.len()); + for s in slots { + match s { + Slot::Var(v) => { + if out.contains(v) { + return None; + } + out.push(*v); + } + Slot::Const(_) => return None, + } + } + Some(out) +} + +/// GYO acyclicity test on the body hypergraph (atoms = hyperedges, vars = +/// vertices). Returns `true` iff the query is CYCLIC (GYO does not reduce to +/// empty). `atom_vars[i]` is atom i's vertex set. +fn is_cyclic_cq(atom_vars: &[Vec]) -> bool { + // Work on owned mutable vertex sets; `removed[i]` drops an absorbed edge. + let mut edges: Vec> = atom_vars + .iter() + .map(|vs| vs.iter().copied().collect()) + .collect(); + let mut removed = vec![false; edges.len()]; + + loop { + let mut progress = false; + + // (a) Ear removal: drop a vertex that appears in <= 1 live edge. + let mut vert_count: HashMap = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + if removed[i] { + continue; + } + for &v in e { + *vert_count.entry(v).or_insert(0) += 1; + } + } + let ears: Vec = vert_count + .iter() + .filter(|&(_, &c)| c <= 1) + .map(|(&v, _)| v) + .collect(); + if !ears.is_empty() { + for e in edges.iter_mut() { + for v in &ears { + e.remove(v); + } + } + progress = true; + } + + // (b) Absorb an edge that is a subset of another live edge. + let live: Vec = (0..edges.len()).filter(|&i| !removed[i]).collect(); + 'outer: for &i in &live { + if edges[i].is_empty() { + removed[i] = true; + progress = true; + continue; + } + for &j in &live { + if i == j || removed[j] { + continue; + } + if edges[i].is_subset(&edges[j]) { + removed[i] = true; + progress = true; + continue 'outer; + } + } + } + + if !progress { + break; + } + } + + // Cyclic iff any non-empty edge remains (GYO got stuck). + edges + .iter() + .enumerate() + .any(|(i, e)| !removed[i] && !e.is_empty()) +} + +/// Build the general WCOJ plan for `plan`, or `None` if it is not constructible +/// bit-exactly. Splits variables into JOIN vars (in >=2 atoms — the cyclic core) +/// and PAYLOAD vars (in exactly 1 atom — eclass/extra columns). The WCOJ binds +/// each JOIN var one at a time (each being the last-unbound JOIN var of >=1 +/// atom, so that atom keys on already-bound join columns), then recovers each +/// atom's payload via one `propose`. If any driver's join vars cannot be ordered +/// this way, bail (`None` ⇒ binary chain). +fn build_cq_plan(plan: &JoinPlan, atom_vars: &[Vec]) -> Option { + let n_atoms = plan.atoms.len(); + let var_col = &plan.var_col; + let atom_funcs: Vec = plan.atoms.iter().map(|a| a.func).collect(); + + // relation-col lookup: atom i, variable v -> the 0-based slot position. + let atom_var_col = |i: usize, v: u32| -> Option { + plan.atoms[i] + .slots + .iter() + .position(|s| matches!(s, Slot::Var(x) if *x == v)) + }; + + // JOIN vars = vars appearing in >= 2 atoms. PAYLOAD vars = in exactly 1. + let mut var_atom_count: HashMap = HashMap::new(); + for vs in atom_vars { + for &v in vs { + *var_atom_count.entry(v).or_insert(0) += 1; + } + } + let is_join = |v: u32| -> bool { var_atom_count.get(&v).copied().unwrap_or(0) >= 2 }; + let join_vars_of = |i: usize| -> Vec { + atom_vars[i] + .iter() + .copied() + .filter(|&v| is_join(v)) + .collect() + }; + let all_join_vars: HashSet = atom_vars + .iter() + .flatten() + .copied() + .filter(|&v| is_join(v)) + .collect(); + + let mut drivers: Vec = Vec::with_capacity(n_atoms); + for driver_idx in 0..n_atoms { + // Seed: the driver atom's own variables (join + payload), straight from + // the delta row — they are all immediately bound. + let mut bound: HashSet = atom_vars[driver_idx].iter().copied().collect(); + let seed: Vec<(usize, usize)> = atom_vars[driver_idx] + .iter() + .map(|&v| (var_col[&v], atom_var_col(driver_idx, v).unwrap())) + .collect(); + + // Bind JOIN vars one at a time. The next join var must be the LAST + // unbound JOIN var of >= 1 atom (payload vars never block — they are + // recovered after, not used as keys). Prefer the var constrained by the + // MOST atoms (most-constrained-variable = the deepest WCOJ + // intersection); ties broken by var id for determinism. + let mut steps: Vec = Vec::new(); + while bound.iter().filter(|&&v| is_join(v)).count() < all_join_vars.len() { + let mut best: Option<(u32, Vec)> = None; + for &v in &all_join_vars { + if bound.contains(&v) { + continue; + } + let mut ready_atoms: Vec = Vec::new(); + for (i, vs) in atom_vars.iter().enumerate() { + if !vs.contains(&v) { + continue; + } + // v is bindable from atom i iff every OTHER JOIN var of atom + // i is already bound. + if vs + .iter() + .all(|&w| w == v || !is_join(w) || bound.contains(&w)) + { + ready_atoms.push(i); + } + } + if ready_atoms.is_empty() { + continue; + } + let better = match &best { + None => true, + Some((bv, ba)) => { + (ready_atoms.len(), std::cmp::Reverse(v)) + > (ba.len(), std::cmp::Reverse(*bv)) + } + }; + if better { + best = Some((v, ready_atoms)); + } + } + + let (v, ready_atoms) = best?; // no ready join var ⇒ not constructible. + let exts: Vec = ready_atoms + .iter() + .map(|&i| { + // Key on the atom's OTHER join vars (all bound by now). + let key_vars: Vec = + join_vars_of(i).into_iter().filter(|&w| w != v).collect(); + CqExtender { + atom_idx: i, + key_cols: key_vars.iter().map(|w| var_col[w]).collect(), + key_atom_cols: key_vars + .iter() + .map(|&w| atom_var_col(i, w).unwrap()) + .collect(), + propose_col: atom_var_col(i, v).unwrap(), + alt: i < driver_idx, + } + }) + .collect(); + steps.push(CqStep { + bind_col: var_col[&v], + exts, + }); + bound.insert(v); + } + + // Recover each NON-driver atom's payload vars (those appearing only in + // it), keyed on the atom's join columns (all bound). The driver atom's + // payload is already in the seed. An atom with NO payload var still must + // be VALIDATED so its tuple constrains the result; we emit a recover + // with an empty payload (a pure existence `propose`). That is sound (no + // over-count): empty payload ⟺ every column is a join var ⟺ the key + // covers the whole row, and rows are set-semantic, so the key matches at + // most one row (weight 1). In the term encoding every atom carries an + // eclass + extra column, so this empty-payload case does not arise. + let mut recovers: Vec = Vec::new(); + for (i, avs) in atom_vars.iter().enumerate() { + if i == driver_idx { + continue; + } + let payload_vars: Vec = avs.iter().copied().filter(|&v| !is_join(v)).collect(); + let key_vars = join_vars_of(i); + recovers.push(CqRecover { + atom_idx: i, + key_cols: key_vars.iter().map(|w| var_col[w]).collect(), + key_atom_cols: key_vars + .iter() + .map(|&w| atom_var_col(i, w).unwrap()) + .collect(), + payload: payload_vars + .iter() + .map(|&v| (var_col[&v], atom_var_col(i, v).unwrap())) + .collect(), + alt: i < driver_idx, + }); + } + + drivers.push(CqDriver { + driver_idx, + seed, + steps, + recovers, + }); + } + + Some(CqPlan { + atom_funcs, + drivers, + }) +} + +/// Detect a general WCOJ-worthy body: a >=3-atom CYCLIC query for which a +/// bit-exact WCOJ plan is constructible. Returns the plan, or `None` (stay on +/// the binary chain). See the module note above for the criteria. +pub(crate) fn detect_cyclic_cq(plan: &JoinPlan, allow_acyclic: bool) -> Option { + let trace = std::env::var_os("FLOWLOG_WCOJ_TRACE").is_some(); + if plan.atoms.len() < 3 { + return None; + } + // All atoms must be pure distinct-Var atoms (no const/repeated-var + // selections — those stay on the binary path). + let mut atom_vars: Vec> = Vec::with_capacity(plan.atoms.len()); + for a in &plan.atoms { + match atom_distinct_vars(&a.slots) { + Some(vs) => atom_vars.push(vs), + None => { + #[allow(clippy::disallowed_macros)] + if trace { + eprintln!( + "[WCOJ] detect_cyclic_cq: bail (atom with const/repeated var) atoms={}", + plan.atoms.len() + ); + } + return None; + } + } + } + // Only fire where the binary chain provably blows up: a CYCLIC join. + let cyclic = is_cyclic_cq(&atom_vars); + #[allow(clippy::disallowed_macros)] + if trace { + eprintln!( + "[WCOJ] detect_cyclic_cq: atoms={} cyclic={cyclic} atom_vars={atom_vars:?}", + plan.atoms.len() + ); + } + // Broadening WCOJ to acyclic >=3-atom rules is off by default (callers + // pass `allow_acyclic: false`). Tests that verify acyclic correctness + // pass `true` explicitly. + if !cyclic && !allow_acyclic { + return None; + } + let plan_out = build_cq_plan(plan, &atom_vars); + #[allow(clippy::disallowed_macros)] + if trace && plan_out.is_none() { + eprintln!("[WCOJ] detect_cyclic_cq: cyclic but NOT constructible -> binary chain"); + } + plan_out +} + +/// The persistent, in-process DD body join for one rule. Built once; driven +/// across epochs by [`step`]. Owns its timely `Worker`, so it can be stepped +/// between host iterations without re-spawning threads. +pub struct PersistentDdJoin { + worker: Worker, + /// One input session per atom occurrence (in `plan.atoms` order). + inputs: Vec>, + /// Probe on the output, to `step_while` until the epoch is fully processed. + probe: ProbeHandle, + /// Shared capture buffer: the output binding deltas of the most-recent + /// epoch, drained by [`step`]. `inspect_batch` appends `(row, weight)`. + captured: Rc>>, + /// `func` -> atom-occurrence indices reading it (self-join fan-out). + occ_of_func: HashMap>, + /// Number of canonical body variables (binding-row width in use). + n_vars: usize, + /// Current epoch (monotonic; advanced once per [`step`]). + epoch: u32, +} + +impl PersistentDdJoin { + /// Build the persistent DD dataflow for `plan` ONCE. + pub fn build(plan: &JoinPlan) -> Result { + let alloc = Allocator::Thread(Thread::default()); + let mut worker = Worker::new( + WorkerConfig::default(), + alloc, + Some(std::time::Instant::now()), + ); + if prof_enabled() { + PROF_WORKERS.fetch_add(1, Ordering::Relaxed); + PROF_INPUT_SESSIONS.fetch_add(plan.atoms.len() as u64, Ordering::Relaxed); + } + + let n_atoms = plan.atoms.len(); + let atom_slots: Vec> = plan.atoms.iter().map(|a| a.slots.clone()).collect(); + let var_col = plan.var_col.clone(); + let n_vars = plan.var_order.len(); + let captured: Rc>> = Rc::new(RefCell::new(Vec::new())); + let captured_in = Rc::clone(&captured); + let probe: ProbeHandle = ProbeHandle::new(); + let probe_in = probe.clone(); + + let inputs = worker.dataflow::(|scope| { + let mut inputs: Vec> = Vec::with_capacity(n_atoms); + let mut collections = Vec::with_capacity(n_atoms); + for _ in 0..n_atoms { + let mut session: InputSession = InputSession::new(); + let coll = session.to_collection(scope); + inputs.push(session); + collections.push(coll); + } + + // Left-deep join. Start from atom 0's bindings (map each relation row + // into a canonical binding row, dropping rows whose const / repeated- + // var constraints fail). Then join successive atoms on shared bound + // columns. `bound[c]` tracks which canonical columns are filled. + let mut bound = vec![false; n_vars]; + + let slots0 = atom_slots[0].clone(); + let vc0 = var_col.clone(); + let mut cur = collections[0] + .clone() + .flat_map(move |r: Row| bind_atom(&r, &slots0, &vc0)); + mark_bound(&atom_slots[0], &var_col, &mut bound); + + for i in 1..n_atoms { + let slots = atom_slots[i].clone(); + // Shared variables = atom vars already bound by a previous atom. + let shared: Vec = atom_vars(&slots) + .into_iter() + .filter(|v| var_col.get(v).map(|&c| bound[c]).unwrap_or(false)) + .collect(); + + // Left side: key the current bindings by the shared canonical + // columns. Right side: key the atom rows by the matching slot + // positions. `.join` then yields `(key, (bind, relrow))` pairs. + let shared_cols_left: Vec = shared.iter().map(|v| var_col[v]).collect(); + let shared_atom_cols: Vec = shared + .iter() + .map(|v| { + slots + .iter() + .position(|s| matches!(s, Slot::Var(x) if x == v)) + .expect("shared var present in atom") + }) + .collect(); + + let scl = shared_cols_left.clone(); + let left = cur.map(move |b: Row| (pack_key(&b, &scl), b)); + let sac = shared_atom_cols.clone(); + let right = collections[i] + .clone() + .map(move |r: Row| (pack_key(&r, &sac), r)); + + let slotsc = slots.clone(); + let vc = var_col.clone(); + let bound_now = bound.clone(); + cur = left + .join(right) + .flat_map(move |(_k, (b, r)): (Key, (Row, Row))| { + merge_atom_into(&b, &r, &slotsc, &vc, &bound_now) + }); + + mark_bound(&slots, &var_col, &mut bound); + } + + // `.distinct()` makes the binding set set-semantic (weights ±1), + // matching egglog relations. `.consolidate()` collapses any + // multiplicities before capture. + let out = cur + .map(|b: Row| (b, ())) + .distinct() + .map(|(b, ())| b) + .consolidate(); + + // Capture the per-epoch output delta into the shared buffer via the + // raw timely stream's batch inspector (DD `inspect_batch` gives us + // `&[(Row, time, isize)]`). We DON'T integrate — each epoch we read + // exactly the delta produced at that epoch's time. + let cap = Rc::clone(&captured_in); + out.inner + .inspect_batch(move |_t, batch| { + let mut buf = cap.borrow_mut(); + for (row, _time, w) in batch.iter() { + buf.push((*row, *w)); + } + }) + .probe_with(&probe_in); + + inputs + }); + + let mut occ_of_func: HashMap> = HashMap::new(); + for (ord, a) in plan.atoms.iter().enumerate() { + occ_of_func.entry(a.func).or_default().push(ord); + } + + Ok(PersistentDdJoin { + worker, + inputs, + probe, + captured, + occ_of_func, + n_vars, + epoch: 0, + }) + } + + /// Feed one epoch of signed relation deltas, advance the timestamp, run the + /// worker to this epoch's fixpoint, and return the resulting binding delta as + /// `(binding_row_as_var_order_vec, weight)`. + /// + /// CRUCIAL: the InputSessions are NEVER cleared — only the delta is pushed, + /// so the DD arrangements persist and the join is genuinely incremental. + pub fn step( + &mut self, + deltas: &HashMap, isize)>>, + ) -> Result, isize)>> { + let prof = prof_enabled(); + let t_feed = std::time::Instant::now(); + let mut pushed = false; + for (func, rows) in deltas { + if let Some(occs) = self.occ_of_func.get(func) { + for &ord in occs { + for (row, w) in rows { + self.inputs[ord].update(pack_row(row), *w); + pushed = true; + } + } + } + } + // No input change for this rule's body ⇒ no new bindings: skip the epoch + // entirely (still advance the logical clock so a later real delta is + // ordered after). This short-circuits the many no-op rebuild re-runs. + let next_epoch = self.epoch + 1; + if !pushed { + self.epoch = next_epoch; + return Ok(Vec::new()); + } + + self.captured.borrow_mut().clear(); + + // Advance every input to the next epoch and flush, then step the worker + // until the probe frontier passes the epoch we just closed — i.e. all + // output for this epoch has been produced. + for inp in &mut self.inputs { + inp.advance_to(next_epoch); + inp.flush(); + } + if prof { + add_ns(&PROF_FEED_NS, t_feed.elapsed()); + PROF_STEP_CALLS.fetch_add(1, Ordering::Relaxed); + } + let t_step = std::time::Instant::now(); + let probe = self.probe.clone(); + self.worker.step_while(|| probe.less_than(&next_epoch)); + if prof { + add_ns(&PROF_STEP_NS, t_step.elapsed()); + } + self.epoch = next_epoch; + + // Drain the capture buffer; consolidate duplicate rows that may have been + // emitted across multiple internal steps within the epoch. + let mut acc: HashMap, isize> = HashMap::new(); + for (row, w) in self.captured.borrow_mut().drain(..) { + let key: Vec = (0..self.n_vars).map(|i| row[i]).collect(); + *acc.entry(key).or_insert(0) += w; + } + let out: Vec<(Vec, isize)> = acc.into_iter().filter(|(_, w)| *w != 0).collect(); + // SPIKE evidence (gated `FLOWLOG_DD_NATIVE_TRACE`): per-epoch INPUT delta + // size vs OUTPUT binding-delta size. Incrementality shows up as: output + // tracks the input delta, NOT the (growing) integral — epoch K does only + // delta·integral work, never a full recompute. The `pos`/`neg` split shows + // retraction (negative-weight outputs) flowing through DD's signed weights. + if trace_enabled() { + let in_n: usize = deltas.values().map(|v| v.len()).sum(); + let pos = out.iter().filter(|(_, w)| *w > 0).count(); + let neg = out.iter().filter(|(_, w)| *w < 0).count(); + #[allow(clippy::disallowed_macros)] + { + eprintln!( + "[dd_native] epoch {} : input_delta_rows={} -> output_binding_delta pos={} neg={}", + next_epoch, in_n, pos, neg + ); + } + } + Ok(out) + } +} + +// =========================================================================== +// FusedDdJoin — ONE shared worker + ONE dataflow per RULESET +// =========================================================================== +// +// `PersistentDdJoin` builds one timely `Worker` PER RULE: a program with R +// atom-bearing rules spins up R independent workers, and a body relation read by +// K rules gets K separate `InputSession`s (each fed the same delta) + K separate +// arrangements, each stepped to fixpoint separately. Step-0 profiling +// (`FLOWLOG_DD_PROF`) on math-microbenchmark N=9 showed 79 workers, 173 +// InputSessions, and 1.996s of 2.43s total spent inside `worker.step_while` — the +// per-rule-worker duplication is the dominant cost. +// +// `FusedDdJoin` collapses this to ONE worker hosting ONE `worker.dataflow(...)` +// scope for the whole ruleset (keyed by the sorted live rule-index list, exactly +// like feldera's `FusedJoin`). Within that scope: +// - every DISTINCT body relation across all rules gets ONE `InputSession` → +// ONE base `Collection`, `.distinct()`'d ONCE and SHARED by every atom +// occurrence (in every rule) that reads it. Cloning a DD collection is a +// handle copy, so the shared `.distinct()` arrangement is built once, not K +// times — the dedup win. +// - each rule is a left-deep join sub-stream reading those shared collections, +// with its OWN `inspect_batch` capture into a per-rule `Rc>`. +// Per epoch: feed each relation's delta ONCE into its shared input, advance + +// step the SINGLE worker once, then drain each rule's capture buffer. The +// host-side prim re-run + `apply_head` are unchanged (the caller does them). +// +// The NEVER-CLEAR / fed-only-deltas invariant is preserved (the InputSessions +// persist across epochs = genuinely incremental), as is the external epoch-drive. + +/// A fused, delta-fed body join for a WHOLE ruleset on a single shared timely +/// `Worker`. Built once via [`FusedDdJoin::build`]; driven across epochs via +/// [`FusedDdJoin::step`] with a SINGLE `worker.step_while` per call. +pub struct FusedDdJoin { + worker: Worker, + /// One shared input session per DISTINCT body relation across all rules. + inputs: HashMap>, + /// Single probe on all rule outputs (they share the dataflow scope, so one + /// probe gates the whole epoch's fixpoint). + probe: ProbeHandle, + /// The fused rules, in build (= sorted rule-index) order. Each carries its + /// own capture buffer + var width. + rules: Vec, + /// Current epoch (monotonic; advanced once per [`step`]). + epoch: u32, +} + +/// One rule's lowering inside a [`FusedDdJoin`]: its rule index (for routing +/// bindings to its head), its per-epoch output capture buffer, and its width. +struct FusedRule { + idx: usize, + /// This rule's per-epoch output binding-delta capture (`inspect_batch` + /// appends `(row, weight)`; drained by [`FusedDdJoin::step`]). + captured: Rc>>, + /// Number of canonical body variables (binding-row width in use). + n_vars: usize, + /// `func` -> atom-occurrence indices reading it within THIS rule (self-join + /// fan-out is handled at build time via the shared collection, so this is + /// only used to know which relations this rule reads). + body_funcs: Vec, +} + +impl FusedDdJoin { + /// Build ONE worker + ONE dataflow for the whole ruleset. `plans` pairs each + /// rule's index with its [`JoinPlan`], in the order they should fire. EVERY + /// rule — congruence, user, and `@rebuild_rule*` canonicalization — runs + /// through the same general fused join (the sound `step_symmetric` path). + pub fn build( + plans: &[(usize, JoinPlan)], + wcoj: bool, + allow_acyclic: bool, + ) -> Result { + let alloc = Allocator::Thread(Thread::default()); + let mut worker = Worker::new( + WorkerConfig::default(), + alloc, + Some(std::time::Instant::now()), + ); + if prof_enabled() { + PROF_WORKERS.fetch_add(1, Ordering::Relaxed); + } + + // Distinct body relations across all rules → one shared input each. + let mut funcs: Vec = Vec::new(); + for (_, plan) in plans { + for a in &plan.atoms { + if !funcs.contains(&a.func) { + funcs.push(a.func); + } + } + } + if prof_enabled() { + PROF_INPUT_SESSIONS.fetch_add(funcs.len() as u64, Ordering::Relaxed); + } + + // Owned per-rule plan snapshots so the `move` dataflow closure is 'static. + struct RulePlan { + idx: usize, + atoms: Vec>, + atom_funcs: Vec, + var_col: HashMap, + n_vars: usize, + body_funcs: Vec, + /// `--wcoj` and this rule is the recognized triangle ⇒ build the WCOJ + /// delta query instead of the binary `.join` chain. `None` ⇒ binary. + triangle: Option, + /// `--wcoj` and this rule is a general detected cyclic >=3-atom body + /// (NOT already matched by `triangle`) ⇒ the general WCOJ delta + /// query. `None` ⇒ triangle or binary. + cq: Option, + /// `FLOWLOG_PROJECT`: per-step column-reuse layout ⇒ the binary chain + /// runs with projecting merges and packs the captured row to the + /// surviving vars. `None` ⇒ the historical static-column chain. + projection: Option, + } + let rule_plans: Vec = plans + .iter() + .map(|(idx, plan)| { + let atom_funcs: Vec = plan.atoms.iter().map(|a| a.func).collect(); + let mut body_funcs: Vec = Vec::new(); + for &f in &atom_funcs { + if !body_funcs.contains(&f) { + body_funcs.push(f); + } + } + // Detect WCOJ shapes ONLY when `--wcoj` is set; off ⇒ both + // `None`, so the build is byte-identical to the binary path. The + // Stage-1 triangle is kept as its own hand-built path (the + // regression guarantee); other detected cyclic >=3-atom bodies + // route to the general construction. A rule matched by the + // triangle never also takes the general path. + let triangle = if wcoj { detect_triangle(plan) } else { None }; + let cq = if wcoj && triangle.is_none() { + detect_cyclic_cq(plan, allow_acyclic) + } else { + None + }; + // With projection the CAPTURED row is packed to the surviving + // vars (columns `0..head_vars.len()`), so the capture/scatter + // width is `head_vars.len()`, not the full static var count. + let n_vars = match &plan.projection { + Some(p) => p.head_vars.len(), + None => plan.var_order.len(), + }; + RulePlan { + idx: *idx, + atoms: plan.atoms.iter().map(|a| a.slots.clone()).collect(), + atom_funcs, + var_col: plan.var_col.clone(), + n_vars, + body_funcs, + triangle, + cq, + projection: plan.projection.clone(), + } + }) + .collect(); + + let probe: ProbeHandle = ProbeHandle::new(); + let probe_in = probe.clone(); + // Per-rule capture buffers, allocated outside the closure so we can keep a + // clone here and route each rule's output to its head after `step`. + let captures: Vec>>> = rule_plans + .iter() + .map(|_| Rc::new(RefCell::new(Vec::new()))) + .collect(); + let captures_in = captures.clone(); + let funcs_in = funcs.clone(); + // The per-rule metadata `FusedRule` needs (kept here; the closure consumes + // `rule_plans` for the dataflow build). + let rule_meta: Vec<(usize, usize, Vec)> = rule_plans + .iter() + .map(|rp| (rp.idx, rp.n_vars, rp.body_funcs.clone())) + .collect(); + + // PERF: the per-epoch input delta is already set-semantic — it is built + // from a `HashSet` set-difference vs the fed view (`interpret::fused_bindings`), + // so each row appears at most once with weight ±1. The input integral + // therefore stays 0/1 per row WITHOUT `.distinct()`, making the input + // distinct (a full integral + per-key consolidation every epoch, over the + // LARGE relation integrals) pure overhead. Dropped by default; set + // `FLOWLOG_DD_KEEP_INPUT_DISTINCT` to restore it. (Mirrors feldera's + // `FELDERA_KEEP_INPUT_DISTINCT` finding.) + let keep_input_distinct = std::env::var_os("FLOWLOG_DD_KEEP_INPUT_DISTINCT").is_some(); + let keep_output_distinct = std::env::var_os("FLOWLOG_DD_KEEP_OUTPUT_DISTINCT").is_some(); + let inputs = worker.dataflow::(move |scope| { + // ONE shared input + base collection per distinct relation, shared by + // every atom occurrence (in every rule) that reads it. + let mut inputs: HashMap> = HashMap::new(); + let mut rel_coll: HashMap = HashMap::new(); + for &f in &funcs_in { + let mut session: InputSession = InputSession::new(); + let base = session.to_collection(scope); + let coll = if keep_input_distinct { + base.map(|r: Row| (r, ())).distinct().map(|(r, ())| r) + } else { + base + }; + inputs.insert(f, session); + rel_coll.insert(f, coll); + } + + for (rp, cap) in rule_plans.iter().zip(captures_in.iter()) { + // This rule's per-atom collection vector, from the SHARED relation + // collections (cloning a DD collection is just a handle copy). + let n_atoms = rp.atoms.len(); + let atom_slots = &rp.atoms; + let var_col = &rp.var_col; + let n_vars = rp.n_vars; + + // `--wcoj`: the recognized triangle rule runs the worst-case- + // optimal delta query (prefix-extension + AltNeu 3-stream + // decomposition) over the SHARED Mul/Add input collections, + // emitting the SAME n_vars-wide binding Row as the binary chain. + let cur = if let Some(tri) = &rp.triangle { + // Diagnostic (gated `FLOWLOG_WCOJ_TRACE`): confirm which rules + // route to the WCOJ path. Mirrors the other `FLOWLOG_*` debug + // gates; zero cost when off. + #[allow(clippy::disallowed_macros)] + if std::env::var_os("FLOWLOG_WCOJ_TRACE").is_some() { + eprintln!( + "[WCOJ] triangle rule idx={} mul={:?} add={:?}", + rp.idx, tri.mul_func, tri.add_func + ); + } + let mul = rel_coll[&tri.mul_func].clone(); + let add = rel_coll[&tri.add_func].clone(); + wcoj_triangle_collection(scope, mul, add, tri.clone()) + } else if let Some(cq) = &rp.cq { + // GENERAL WCOJ: a detected cyclic >=3-atom body. One shared + // input collection per atom (in atom order), fed to the + // k-stream delta decomposition. Emits the same n_vars-wide + // binding Row as the binary chain. + #[allow(clippy::disallowed_macros)] + if std::env::var_os("FLOWLOG_WCOJ_TRACE").is_some() { + eprintln!( + "[WCOJ] cyclic-cq rule idx={} atoms={} funcs={:?}", + rp.idx, + cq.atom_funcs.len(), + cq.atom_funcs + ); + } + let colls: Vec<_> = cq.atom_funcs.iter().map(|f| rel_coll[f].clone()).collect(); + wcoj_cq_collection(scope, &colls, cq.clone()) + } else if let Some(proj) = &rp.projection { + // PROJECTED binary chain (`FLOWLOG_PROJECT`): the binding row + // is laid out per-step with column REUSE (`proj.step_col`), so + // a rule whose static var count exceeds `W` still fits. The + // chain is otherwise identical to the static path below; the + // only differences are (a) every var's column is the per-step + // column instead of the static `var_col`, (b) each merge REMAPS + // carried-over vars whose column changed (reuse) and zeroes + // freed columns, and (c) a final `.map` packs the surviving + // vars into columns `0..head_vars.len()` for the capture. + let step_col = &proj.step_col; + let slots0 = atom_slots[0].clone(); + let sc0 = step_col[0].clone(); + let mut cur = rel_coll[&rp.atom_funcs[0]] + .clone() + .flat_map(move |r: Row| bind_atom(&r, &slots0, &sc0)); + + for i in 1..n_atoms { + let slots = atom_slots[i].clone(); + let prev = &step_col[i - 1]; + let curl = &step_col[i]; + // Shared = atom vars already live (present in `prev`). + let shared: Vec = atom_vars(&slots) + .into_iter() + .filter(|v| prev.contains_key(v)) + .collect(); + // Left key reads each shared var from its PREVIOUS column. + let shared_cols_left: Vec = shared.iter().map(|v| prev[v]).collect(); + let shared_atom_cols: Vec = shared + .iter() + .map(|v| { + slots + .iter() + .position(|s| matches!(s, Slot::Var(x) if x == v)) + .expect("shared var present in atom") + }) + .collect(); + + let scl = shared_cols_left.clone(); + let left = cur.map(move |b: Row| (pack_key(&b, &scl), b)); + let sac = shared_atom_cols.clone(); + let right = rel_coll[&rp.atom_funcs[i]] + .clone() + .map(move |r: Row| (pack_key(&r, &sac), r)); + + // Carried vars: live at BOTH prev and cur but NOT produced + // by this atom — copy them from prev[v] to cur[v]. Atom + // vars: written/validated at cur[v]. (prev_col, cur_col, + // is_shared, atom_col) per relevant var, precomputed. + let slotsc = slots.clone(); + let prevc = prev.clone(); + let curc = curl.clone(); + cur = left + .join(right) + .flat_map(move |(_k, (b, r)): (Key, (Row, Row))| { + remap_merge_atom_into(&b, &r, &slotsc, &prevc, &curc) + }); + } + + // Final projection: pack the surviving vars (at their last-step + // columns `head_cols`) into the capture columns `0..k`, zeroing + // the rest, so the head scatter reads `bind[i]` for surviving + // var `i` exactly like the static path. + let head_cols = proj.head_cols.clone(); + cur.map(move |b: Row| pack_key(&b, &head_cols)) + } else { + let mut bound = vec![false; n_vars]; + let slots0 = atom_slots[0].clone(); + let vc0 = var_col.clone(); + let mut cur = rel_coll[&rp.atom_funcs[0]] + .clone() + .flat_map(move |r: Row| bind_atom(&r, &slots0, &vc0)); + mark_bound(&atom_slots[0], var_col, &mut bound); + + for i in 1..n_atoms { + let slots = atom_slots[i].clone(); + let shared: Vec = atom_vars(&slots) + .into_iter() + .filter(|v| var_col.get(v).map(|&c| bound[c]).unwrap_or(false)) + .collect(); + let shared_cols_left: Vec = + shared.iter().map(|v| var_col[v]).collect(); + let shared_atom_cols: Vec = shared + .iter() + .map(|v| { + slots + .iter() + .position(|s| matches!(s, Slot::Var(x) if x == v)) + .expect("shared var present in atom") + }) + .collect(); + + let scl = shared_cols_left.clone(); + let left = cur.map(move |b: Row| (pack_key(&b, &scl), b)); + let sac = shared_atom_cols.clone(); + let right = rel_coll[&rp.atom_funcs[i]] + .clone() + .map(move |r: Row| (pack_key(&r, &sac), r)); + + let slotsc = slots.clone(); + let vc = var_col.clone(); + let bound_now = bound.clone(); + cur = left + .join(right) + .flat_map(move |(_k, (b, r)): (Key, (Row, Row))| { + merge_atom_into(&b, &r, &slotsc, &vc, &bound_now) + }); + mark_bound(&slots, var_col, &mut bound); + } + cur + }; + + // PERF: the output `.distinct()` is redundant here. `step` + // accumulates each rule's binding deltas into a per-key weight map + // and `interpret::fused_bindings` inspects only the SIGN of the net + // weight (>0 ⇒ one env; net-zero already filtered). distinct would + // clamp the binding multiplicity to {0,1}, but since only the sign + // is observed and net-zero rows are dropped, the clamp is + // unobservable. `.consolidate()` still collapses per-key + // multiplicities so the captured batch is one signed row per key. + // Dropped by default; set `FLOWLOG_DD_KEEP_OUTPUT_DISTINCT` to + // restore. (Mirrors feldera's `FELDERA_KEEP_OUTPUT_DISTINCT`.) + let consolidated = if keep_output_distinct { + cur.map(|b: Row| (b, ())) + .distinct() + .map(|(b, ())| b) + .consolidate() + } else { + cur.consolidate() + }; + let out = consolidated; + + let cap = Rc::clone(cap); + out.inner + .inspect_batch(move |_t, batch| { + let mut buf = cap.borrow_mut(); + for (row, _time, w) in batch.iter() { + buf.push((*row, *w)); + } + }) + .probe_with(&probe_in); + } + + inputs + }); + + let rules: Vec = rule_meta + .into_iter() + .zip(captures) + .map(|((idx, n_vars, body_funcs), captured)| FusedRule { + idx, + captured, + n_vars, + body_funcs, + }) + .collect(); + + Ok(FusedDdJoin { + worker, + inputs, + probe, + rules, + epoch: 0, + }) + } + + /// The rule indices this fused worker serves (build order). + pub fn rule_indices(&self) -> Vec { + self.rules.iter().map(|r| r.idx).collect() + } + + /// The body relations the fused rule at build position `pos` reads. + pub fn rule_body_funcs(&self, pos: usize) -> &[FunctionId] { + &self.rules[pos].body_funcs + } + + /// Feed one epoch of signed relation deltas into the SHARED inputs, advance + /// the timestamp, run the SINGLE worker to this epoch's fixpoint, and return + /// per-rule binding deltas. The outer `Vec` is in [`rule_indices`] order; each + /// inner `Vec` is `(binding_row_as_var_order_vec, weight)`. + /// + /// CRUCIAL: the InputSessions are NEVER cleared — only the delta is pushed, so + /// the DD arrangements persist and the join is genuinely incremental. + pub fn step( + &mut self, + deltas: &HashMap, isize)>>, + ) -> Result, isize)>>> { + // EVERY rule — congruence, user, and `@rebuild_rule*` canonicalization — + // runs through the SAME symmetric incremental join: feed ALL deltas, one + // sub-step, capture every rule. FlowLog is on the fast relational + // term-encoding, so the rebuild rules take no special path (the unsound + // δUF-driven / transient-pulse rebuild variants have been deleted). + self.step_symmetric(deltas) + } + + /// Step the SINGLE worker to `epoch`, accounting feed/step time. + fn drive_to(&mut self, epoch: u32, t_feed: std::time::Instant, prof: bool) { + for inp in self.inputs.values_mut() { + inp.advance_to(epoch); + inp.flush(); + } + if prof { + add_ns(&PROF_FEED_NS, t_feed.elapsed()); + PROF_STEP_CALLS.fetch_add(1, Ordering::Relaxed); + } + let t_step = std::time::Instant::now(); + let probe = self.probe.clone(); + self.worker.step_while(|| probe.less_than(&epoch)); + if prof { + add_ns(&PROF_STEP_NS, t_step.elapsed()); + } + } + + /// Drain each rule's capture buffer; keep a rule's output into `accs` only if + /// `keep(rule)` (route a sub-step's output to the rules it is meant for). + fn drain_into(&self, accs: &mut [HashMap, isize>], keep: impl Fn(&FusedRule) -> bool) { + for (rule, acc) in self.rules.iter().zip(accs.iter_mut()) { + let drained: Vec<(Row, isize)> = rule.captured.borrow_mut().drain(..).collect(); + if !keep(rule) { + continue; + } + for (row, w) in drained { + let key: Vec = (0..rule.n_vars).map(|i| row[i]).collect(); + *acc.entry(key).or_insert(0) += w; + } + } + } + + /// The original symmetric incremental join: feed ALL deltas, one sub-step, + /// capture every rule. Behavior-identical to the pre-Phase-2 `step`. + fn step_symmetric( + &mut self, + deltas: &HashMap, isize)>>, + ) -> Result, isize)>>> { + let prof = prof_enabled(); + let t_feed = std::time::Instant::now(); + let mut pushed = false; + for (func, rows) in deltas { + if let Some(inp) = self.inputs.get_mut(func) { + for (row, w) in rows { + inp.update(pack_row(row), *w); + pushed = true; + } + } + } + let next_epoch = self.epoch + 1; + if !pushed { + self.epoch = next_epoch; + return Ok(vec![Vec::new(); self.rules.len()]); + } + + for rule in &self.rules { + rule.captured.borrow_mut().clear(); + } + self.drive_to(next_epoch, t_feed, prof); + self.epoch = next_epoch; + + if trace_enabled() { + let total: usize = self.rules.iter().map(|r| r.captured.borrow().len()).sum(); + let n_rules = self.rules.len(); + use egglog_numeric_id::NumericId; + let funcs: Vec = deltas.keys().map(|f| f.rep()).collect(); + let delta_rows: usize = deltas.values().map(|r| r.len()).sum(); + #[allow(clippy::disallowed_macros)] + { + eprintln!( + "[dd_symmetric] n_rules={n_rules} delta_funcs={funcs:?} delta_rows={delta_rows} total_out={total}" + ); + } + } + + let mut accs: Vec, isize>> = + (0..self.rules.len()).map(|_| HashMap::new()).collect(); + self.drain_into(&mut accs, |_| true); + Ok(accs + .into_iter() + .map(|acc| acc.into_iter().filter(|(_, w)| *w != 0).collect()) + .collect()) + } +} + +/// Pack a slice of column values into a fixed-width row (0-padded). +fn pack_row(vals: &[u32]) -> Row { + let mut a = empty_row(); + for (i, v) in vals.iter().enumerate() { + a[i] = *v; + } + a +} + +/// Build a join key from selected columns (packed into the low slots). +fn pack_key(r: &Row, cols: &[usize]) -> Key { + let mut a = empty_row(); + for (i, &c) in cols.iter().enumerate() { + a[i] = r[c]; + } + a +} + +/// Distinct variables appearing in an atom (column order). +fn atom_vars(slots: &[Slot]) -> Vec { + let mut out = Vec::new(); + for s in slots { + if let Slot::Var(v) = s { + if !out.contains(v) { + out.push(*v); + } + } + } + out +} + +/// Mark the canonical columns of an atom's variables as bound. +fn mark_bound(slots: &[Slot], var_col: &HashMap, bound: &mut [bool]) { + for s in slots { + if let Slot::Var(v) = s { + if let Some(&c) = var_col.get(v) { + bound[c] = true; + } + } + } +} + +/// Match the first atom's relation row against its slots, producing the initial +/// canonical binding row (or empty vec if a const / repeated-var constraint +/// fails). Returns a `Vec` for `flat_map`. +fn bind_atom(r: &Row, slots: &[Slot], var_col: &HashMap) -> Vec { + let mut out = empty_row(); + let mut local: HashMap = HashMap::new(); + for (i, s) in slots.iter().enumerate() { + let val = r[i]; + match s { + Slot::Const(c) => { + if *c != val { + return Vec::new(); + } + } + Slot::Var(v) => { + if let Some(&prev) = local.get(v) { + if prev != val { + return Vec::new(); + } + } else { + local.insert(*v, val); + out[var_col[v]] = val; + } + } + } + } + vec![out] +} + +/// Merge atom row `r` into binding `b`: already-bound columns must agree; +/// previously-unbound atom vars are written. Empty vec on constraint failure. +fn merge_atom_into( + b: &Row, + r: &Row, + slots: &[Slot], + var_col: &HashMap, + bound: &[bool], +) -> Vec { + let mut out = *b; + let mut local: HashMap = HashMap::new(); + for (i, s) in slots.iter().enumerate() { + let val = r[i]; + match s { + Slot::Const(c) => { + if *c != val { + return Vec::new(); + } + } + Slot::Var(v) => { + if let Some(&prev) = local.get(v) { + if prev != val { + return Vec::new(); + } + continue; + } + local.insert(*v, val); + let c = var_col[v]; + if bound[c] { + if out[c] != val { + return Vec::new(); + } + } else { + out[c] = val; + } + } + } + } + vec![out] +} + +/// Projecting merge for the `FLOWLOG_PROJECT` chain. Like [`merge_atom_into`] but +/// rebuilds the row from scratch under a NEW column layout: `prev` is the +/// left-row layout (step `i-1`), `cur` is the output layout (step `i`). Carried +/// vars (live in both layouts but not produced here) are copied `prev[v]→cur[v]`; +/// the atom's vars are validated (shared) or written (fresh) at `cur[v]`; every +/// other output column is left zeroed. This is what reuses freed columns and so +/// keeps the frontier within `W`. Empty vec on a const / repeated-var / shared- +/// var constraint failure (same semantics as the static merge). +fn remap_merge_atom_into( + b: &Row, + r: &Row, + slots: &[Slot], + prev: &HashMap, + cur: &HashMap, +) -> Vec { + let mut out = empty_row(); + // Carry over every still-live var (present in `cur`) that the left row + // already holds (present in `prev`). Atom-fresh vars are absent from `prev`, + // so they are NOT copied here — they are written from `r` below. + for (&v, &pc) in prev { + if let Some(&cc) = cur.get(&v) { + out[cc] = b[pc]; + } + } + let mut local: HashMap = HashMap::new(); + for (i, s) in slots.iter().enumerate() { + let val = r[i]; + match s { + Slot::Const(c) => { + if *c != val { + return Vec::new(); + } + } + Slot::Var(v) => { + if let Some(&prior) = local.get(v) { + if prior != val { + return Vec::new(); + } + continue; + } + local.insert(*v, val); + // Every atom var is live at this step ⇒ present in `cur`. + let cc = cur[v]; + if prev.contains_key(v) { + // Shared (already bound): the carried value must agree. (The + // join key already enforces this for the key columns; this + // also covers shared vars not in the key, if any.) + if out[cc] != val { + return Vec::new(); + } + } else { + // Fresh var born at this atom: write it. + out[cc] = val; + } + } + } + } + vec![out] +} + +// --------------------------------------------------------------------------- +// Stage 2: GENERAL `--wcoj` worst-case-optimal delta query (any cyclic body) +// --------------------------------------------------------------------------- + +/// A WCOJ collection index over the `Row`-keyed encoding used by the general +/// construction: K = the prefix `Row` (with the relevant bound columns filled, +/// others 0), V = the single proposed variable value (`u32`). +type CqIndex<'scope> = CollectionIndex, isize>; + +/// The key-selector closure type shared by every general-WCOJ extender: read +/// `cols` of the prefix `Row` into a fresh key `Row`. Writing it ONCE here (one +/// source location) makes every `extend_using(make_key_selector(...))` the SAME +/// concrete `CollectionExtender` type, so a step's extenders can live in one +/// `Vec` and be passed as a `&mut [&mut dyn PrefixExtender]` slice to `extend`. +fn cq_key_selector(cols: Vec) -> impl Fn(&Row) -> Row + Clone + 'static { + move |p: &Row| { + let mut k = empty_row(); + for (i, &c) in cols.iter().enumerate() { + k[i] = p[c]; + } + k + } +} + +/// Build the GENERAL worst-case-optimal join for a detected cyclic body +/// (`CqPlan`) as a collection of full-width binding `Row`s — the SAME shape the +/// binary `.join` chain emits, so the downstream consolidate / capture / env +/// scatter is bit-identical. +/// +/// Implements the full k-stream delta decomposition (one delta query per body +/// atom) inside ONE `AltNeu` nested scope, exactly mirroring the Stage-1 +/// triangle: in delta query `dQ/dA_d`, atoms with index < d read the ALT (old) +/// trace and atoms with index > d read the NEU (new) trace, so simultaneous +/// cross-atom updates are counted exactly once. ALL reads (the multiway +/// intersections AND the per-atom payload recoveries) are inside the AltNeu +/// scope. Each JOIN step binds ONE join variable: a >=2-extender step is the +/// WCOJ multiway `extend` (the intersection that collapses the intermediate); a +/// 1-extender step is a `propose_using`. After the join core is pinned, each +/// non-driver atom's payload vars are recovered (and the atom validated) by one +/// `propose` keyed on its join columns. +fn wcoj_cq_collection<'scope>( + scope: Scope<'scope, u32>, + colls: &[VecCollection<'scope, u32, Row, isize>], + plan: CqPlan, +) -> VecCollection<'scope, u32, Row, isize> { + use differential_dataflow::AsCollection; + use timely::dataflow::operators::Concatenate; + + scope.scoped::, _, _>("WcojCq", move |inner| { + // Enter every relation; build an ALT and a NEU copy of each. + let alt: Vec<_> = colls.iter().map(|c| c.clone().enter(inner)).collect(); + let neu: Vec<_> = alt + .iter() + .map(|c| c.clone().delay(|t| AltNeu::neu(t.time))) + .collect(); + + // Per-driver delta query. We build the indexes each driver needs lazily + // (cloning a CollectionIndex shares its traces, so repeats are cheap). + let mut streams = Vec::with_capacity(plan.drivers.len()); + for driver in &plan.drivers { + // Seed the prefix from the driver atom's delta row. + let seed = driver.seed.clone(); + let prefix = alt[driver.driver_idx].clone().map(move |r: Row| { + let mut p = empty_row(); + for &(bind_col, rel_col) in &seed { + p[bind_col] = r[rel_col]; + } + p + }); + + let mut cur = prefix; + for step in &driver.steps { + // One index per extender (over the ALT or NEU copy of its atom), + // keyed by the atom's key columns, proposing the step's var. + let mut indexes: Vec = step + .exts + .iter() + .map(|e| { + let src = if e.alt { + alt[e.atom_idx].clone() + } else { + neu[e.atom_idx].clone() + }; + let key_atom_cols = e.key_atom_cols.clone(); + let propose_col = e.propose_col; + // A JOIN-var proposal is SET-semantic: "does this atom hold + // a row with these key cols proposing this var?" — weight 1, + // regardless of how many full rows witness it. Without the + // `.distinct()`, an atom carrying PAYLOAD (or any) columns + // beyond (key, propose) yields the SAME (K,V) pair once per + // such row, and `propose`/`validate` MULTIPLY the prefix + // weight by that multiplicity (the `propose`/`validate` + // traces are non-distinct; only `count` is). That + // over-counted the acyclic star/path bodies (e.g. two F-rows + // sharing (v0,v1) doubled the binding weight). Dedup the + // (K,V) pairs so the proposal is exactly the SET of valid + // extensions. (Payload values are recovered separately, in + // the `recovers` step below, where multiplicity is correct.) + let kv = src + .map(move |r: Row| { + let mut k = empty_row(); + for (i, &c) in key_atom_cols.iter().enumerate() { + k[i] = r[c]; + } + (k, r[propose_col]) + }) + .map(|kv| (kv, ())) + .distinct() + .map(|(kv, ())| kv); + CollectionIndex::index(kv) + }) + .collect(); + + // Build the extenders (all the same concrete type — see + // `cq_key_selector`), then intersect via `extend` (>=2) or + // recover via `propose_using` (==1). + let mut extenders: Vec<_> = indexes + .iter_mut() + .zip(step.exts.iter()) + .map(|(idx, e)| idx.extend_using(cq_key_selector(e.key_cols.clone()))) + .collect(); + + let bind_col = step.bind_col; + cur = if extenders.len() == 1 { + cur.propose_using(&mut extenders[0]) + .map(move |(mut p, val): (Row, u32)| { + p[bind_col] = val; + p + }) + } else { + let mut refs: Vec< + &mut dyn differential_dogs3::PrefixExtender< + AltNeu, + isize, + Prefix = Row, + Extension = u32, + >, + > = extenders + .iter_mut() + .map(|e| { + e as &mut dyn differential_dogs3::PrefixExtender< + AltNeu, + isize, + Prefix = Row, + Extension = u32, + > + }) + .collect(); + cur.extend(&mut refs[..]) + .map(move |(mut p, val): (Row, u32)| { + p[bind_col] = val; + p + }) + }; + } + + // Payload recovery: every non-driver atom is now keyed entirely on + // bound join columns. One `propose` per atom recovers its payload + // vars (V = a Row packing the payload values) AND validates the + // atom's existence (an atom with no payload still must be present). + for rec in &driver.recovers { + let src = if rec.alt { + alt[rec.atom_idx].clone() + } else { + neu[rec.atom_idx].clone() + }; + let key_atom_cols = rec.key_atom_cols.clone(); + let payload = rec.payload.clone(); + // Index (K = join-key Row, V = payload-values Row). + let idx: CollectionIndex, isize> = + CollectionIndex::index(src.map(move |r: Row| { + let mut k = empty_row(); + for (i, &c) in key_atom_cols.iter().enumerate() { + k[i] = r[c]; + } + let mut v = empty_row(); + for (slot, &(_bind, rel_col)) in payload.iter().enumerate() { + v[slot] = r[rel_col]; + } + (k, v) + })); + let mut ext = idx.extend_using(cq_key_selector(rec.key_cols.clone())); + let payload2 = rec.payload.clone(); + cur = cur + .propose_using(&mut ext) + .map(move |(mut p, vals): (Row, Row)| { + for (slot, &(bind_col, _rel)) in payload2.iter().enumerate() { + p[bind_col] = vals[slot]; + } + p + }); + } + + streams.push(cur.inner); + } + + inner.concatenate(streams).as_collection().leave(scope) + }) +} + +// --------------------------------------------------------------------------- +// `--wcoj` triangle worst-case-optimal delta query +// --------------------------------------------------------------------------- + +/// Build the worst-case-optimal triangle join as a differential-dataflow +/// collection of full `n_vars`-wide binding `Row`s — the SAME shape the binary +/// `.join` chain emits, so the downstream consolidate / capture / env scatter +/// is bit-identical. +/// +/// Implements the FULL 3-stream delta decomposition (one delta query per body +/// atom) inside a single `AltNeu` nested scope, following dogsdogsdogs' +/// `examples/delta_query_wcoj.rs`. The total atom order is A0(Mul_a) < +/// A1(Mul_b) < A2(Add): in each delta query, atoms BEFORE the driving atom read +/// the `alt` (old) trace and atoms AFTER read the `neu` (new) trace, so +/// simultaneous cross-atom updates (incl. the Mul self-join) are not +/// double-counted. The ONLY multiway-intersected variable is the core `a` +/// (driven by dA2: intersect the `a`-sets of the two Mul eclasses m1,m2 — the +/// `Σ_a deg(a)²` collapse); every other variable is functionally recovered by a +/// single in-scope `propose_using`. ALL reads (core intersection AND payload +/// recovery) are inside the AltNeu scope — unlike the spike shortcut whose +/// out-of-scope recovery drifted at high iteration counts. +fn wcoj_triangle_collection<'scope>( + scope: Scope<'scope, u32>, + mul: VecCollection<'scope, u32, Row, isize>, + add: VecCollection<'scope, u32, Row, isize>, + tri: TriangleShape, +) -> VecCollection<'scope, u32, Row, isize> { + use differential_dataflow::AsCollection; + use timely::dataflow::operators::Concatenate; + + // Raw-relation column layout (fixed by the term encoding's arity-4 schema): + // Mul row = [arg0=a/c, arg1=b/c-payload, eclass=m, rowextra=x] + // Add row = [m1, m2, eclass=o, rowextra=x] + // (cols 0..3 are the RELATION columns; tri.col_* are the BINDING columns.) + // Each stream assembles a full-width `empty_row()` and writes only the 9 + // triangle columns; columns >= n_vars stay 0 — identical to the binary + // chain, which also leaves unused binding columns at 0. (n_vars itself is + // not needed here; the downstream drain reads cols 0..rule.n_vars.) + let (ca, cb, cm1, cx1) = (tri.col_a, tri.col_b, tri.col_m1, tri.col_x1); + let (cc, cm2, cx2) = (tri.col_c, tri.col_m2, tri.col_x2); + let (co, cx3) = (tri.col_o, tri.col_x3); + + scope.scoped::, _, _>("WcojTriangle", move |inner| { + let mul = mul.enter(inner); + let add = add.enter(inner); + let mul_neu = mul.clone().delay(|t| AltNeu::neu(t.time)); + let add_neu = add.clone().delay(|t| AltNeu::neu(t.time)); + + // --- indices over the entered collections --------------------------- + // Mul indices: (K=eclass m, V=arg a), (K=arg a, V=eclass m), + // (K=(a,m), V=(b,x)) [payload recovery] + let alt_mul_by_m = CollectionIndex::index(mul.clone().map(|r: Row| (r[2], r[0]))); + let alt_mul_by_a = CollectionIndex::index(mul.clone().map(|r: Row| (r[0], r[2]))); + let neu_mul_by_a = CollectionIndex::index(mul_neu.clone().map(|r: Row| (r[0], r[2]))); + let alt_mul_by_am = + CollectionIndex::index(mul.clone().map(|r: Row| ((r[0], r[2]), (r[1], r[3])))); + let neu_mul_by_am = + CollectionIndex::index(mul_neu.clone().map(|r: Row| ((r[0], r[2]), (r[1], r[3])))); + // Add indices: (K=m1, V=m2), (K=m2, V=m1), (K=(m1,m2), V=(o,x)). + let neu_add_by_m1 = CollectionIndex::index(add_neu.clone().map(|r: Row| (r[0], r[1]))); + let neu_add_by_m2 = CollectionIndex::index(add_neu.clone().map(|r: Row| (r[1], r[0]))); + let neu_add_by_m1m2 = + CollectionIndex::index(add_neu.clone().map(|r: Row| ((r[0], r[1]), (r[2], r[3])))); + + // ===== dQ/dA0 : driven by dMul as Mul_a(a,b,m1,x1) ================== + // bound a,b,m1,x1 ; A1(Mul_b) NEU, A2(Add) NEU. + let changes0 = { + // initial prefix: place the Mul delta row's cols into a,b,m1,x1. + let prefix = mul.clone().map(move |r: Row| { + let mut p = empty_row(); + p[ca] = r[0]; + p[cb] = r[1]; + p[cm1] = r[2]; + p[cx1] = r[3]; + p + }); + // 1. intersect m2: Add(m1,·)=m2 (NEU) ∩ Mul(a,·)=m2 (NEU). + let with_m2 = prefix + .extend(&mut [ + &mut neu_add_by_m1.extend_using(move |p: &Row| p[cm1]), + &mut neu_mul_by_a.extend_using(move |p: &Row| p[ca]), + ]) + .map(move |(mut p, m2): (Row, u32)| { + p[cm2] = m2; + p + }); + // 2. recover (c,x2) for Mul_b given (a,m2) (NEU). + let with_cx2 = with_m2 + .propose_using(&mut neu_mul_by_am.extend_using(move |p: &Row| (p[ca], p[cm2]))) + .map(move |(mut p, (c, x2)): (Row, (u32, u32))| { + p[cc] = c; + p[cx2] = x2; + p + }); + // 3. recover (o,x3) for Add given (m1,m2) (NEU). + with_cx2 + .propose_using(&mut neu_add_by_m1m2.extend_using(move |p: &Row| (p[cm1], p[cm2]))) + .map(move |(mut p, (o, x3)): (Row, (u32, u32))| { + p[co] = o; + p[cx3] = x3; + p + }) + }; + + // ===== dQ/dA1 : driven by dMul as Mul_b(a,c,m2,x2) ================== + // bound a,c,m2,x2 ; A0(Mul_a) ALT, A2(Add) NEU. + let changes1 = { + let prefix = mul.clone().map(move |r: Row| { + let mut p = empty_row(); + p[ca] = r[0]; + p[cc] = r[1]; + p[cm2] = r[2]; + p[cx2] = r[3]; + p + }); + // 1. intersect m1: Add(·,m2)=m1 (NEU) ∩ Mul(a,·)=m1 (ALT). + let with_m1 = prefix + .extend(&mut [ + &mut neu_add_by_m2.extend_using(move |p: &Row| p[cm2]), + &mut alt_mul_by_a.extend_using(move |p: &Row| p[ca]), + ]) + .map(move |(mut p, m1): (Row, u32)| { + p[cm1] = m1; + p + }); + // 2. recover (b,x1) for Mul_a given (a,m1) (ALT). + let with_bx1 = with_m1 + .propose_using(&mut alt_mul_by_am.extend_using(move |p: &Row| (p[ca], p[cm1]))) + .map(move |(mut p, (b, x1)): (Row, (u32, u32))| { + p[cb] = b; + p[cx1] = x1; + p + }); + // 3. recover (o,x3) for Add given (m1,m2) (NEU). + with_bx1 + .propose_using(&mut neu_add_by_m1m2.extend_using(move |p: &Row| (p[cm1], p[cm2]))) + .map(move |(mut p, (o, x3)): (Row, (u32, u32))| { + p[co] = o; + p[cx3] = x3; + p + }) + }; + + // ===== dQ/dA2 : driven by dAdd(m1,m2,o,x3) ========================= + // bound m1,m2,o,x3 ; A0(Mul_a) ALT, A1(Mul_b) ALT. + let changes2 = { + let prefix = add.clone().map(move |r: Row| { + let mut p = empty_row(); + p[cm1] = r[0]; + p[cm2] = r[1]; + p[co] = r[2]; + p[cx3] = r[3]; + p + }); + // 1. intersect a: Mul(·,·)=m1 → a (ALT) ∩ Mul(·,·)=m2 → a (ALT). + // THE worst-case-optimal collapse. + let with_a = prefix + .extend(&mut [ + &mut alt_mul_by_m.extend_using(move |p: &Row| p[cm1]), + &mut alt_mul_by_m.extend_using(move |p: &Row| p[cm2]), + ]) + .map(move |(mut p, a): (Row, u32)| { + p[ca] = a; + p + }); + // 2. recover (b,x1) for Mul_a given (a,m1) (ALT). + let with_bx1 = with_a + .propose_using(&mut alt_mul_by_am.extend_using(move |p: &Row| (p[ca], p[cm1]))) + .map(move |(mut p, (b, x1)): (Row, (u32, u32))| { + p[cb] = b; + p[cx1] = x1; + p + }); + // 3. recover (c,x2) for Mul_b given (a,m2) (ALT). + with_bx1 + .propose_using(&mut alt_mul_by_am.extend_using(move |p: &Row| (p[ca], p[cm2]))) + .map(move |(mut p, (c, x2)): (Row, (u32, u32))| { + p[cc] = c; + p[cx2] = x2; + p + }) + }; + + // Concat the three delta streams and leave the AltNeu scope. + let streams = vec![changes0.inner, changes1.inner, changes2.inner]; + inner.concatenate(streams).as_collection().leave(scope) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use egglog_numeric_id::NumericId; + + /// A transitive-closure hop plan `R(x,y), R(y,z)` over one relation, built + /// directly (no full RuleIr). + fn tc_plan(func: FunctionId) -> JoinPlan { + let mut var_col = HashMap::new(); + var_col.insert(0u32, 0usize); + var_col.insert(1u32, 1usize); + var_col.insert(2u32, 2usize); + JoinPlan { + var_order: vec![0, 1, 2], + var_col, + atoms: vec![ + PlanAtom { + func, + slots: vec![Slot::Var(0), Slot::Var(1)], + }, + PlanAtom { + func, + slots: vec![Slot::Var(1), Slot::Var(2)], + }, + ], + projection: None, + } + } + + /// Build a `JoinPlan` directly from `(func, vars)` atoms (all slots are + /// distinct `Var`s — the shape the WCOJ detector accepts). + fn plan_of(atoms: &[(FunctionId, &[u32])]) -> JoinPlan { + let mut var_order: Vec = Vec::new(); + let mut var_col: HashMap = HashMap::new(); + let mut planned: Vec = Vec::new(); + for (func, vars) in atoms { + for &v in *vars { + if !var_col.contains_key(&v) { + var_col.insert(v, var_order.len()); + var_order.push(v); + } + } + planned.push(PlanAtom { + func: *func, + slots: vars.iter().map(|&v| Slot::Var(v)).collect(), + }); + } + JoinPlan { + var_order, + var_col, + atoms: planned, + projection: None, + } + } + + /// Feed one all-at-once delta of `rows` per func into a fresh `FusedDdJoin` + /// (single rule idx 0, plan rebuilt from `atoms`), returning the rule's + /// sorted output bindings. + fn run_once( + atoms: &[(FunctionId, &[u32])], + wcoj: bool, + rows: &HashMap, isize)>>, + ) -> Vec<(Vec, isize)> { + let plan = plan_of(atoms); + let mut j = FusedDdJoin::build(&[(0usize, plan)], wcoj, true).unwrap(); + let mut out = j.step(rows).unwrap().remove(0); + out.sort(); + out + } + + /// ACYCLIC >=3-atom bodies must be bit-exact under the general WCOJ + /// construction (broadened past the cyclic-only gate). We compare the WCOJ + /// output against the binary chain (ground truth) row-for-row, on several + /// acyclic shapes that previously over-derived (cross-product). The WCOJ + /// detector is forced on for the acyclic shapes via `allow_acyclic: true` + /// (production uses `false`; the test helper `run_once` passes `true`). + #[test] + fn wcoj_acyclic_bit_exact_vs_binary() { + let f = FunctionId::new(0); + let g = FunctionId::new(1); + let h = FunctionId::new(2); + + // --- Shape 1: 3-atom STAR F(v0,v1,v2), G(v0,v3), H(v1,v4) ---------- + { + let atoms: &[(FunctionId, &[u32])] = &[(f, &[0, 1, 2]), (g, &[0, 3]), (h, &[1, 4])]; + assert!( + detect_cyclic_cq(&plan_of(atoms), true).is_some(), + "star fires WCOJ" + ); + let mut rows = HashMap::new(); + rows.insert( + f, + vec![ + (vec![1, 2, 100], 1), + (vec![1, 2, 101], 1), + (vec![1, 3, 102], 1), + ], + ); + rows.insert( + g, + vec![(vec![1, 10], 1), (vec![1, 11], 1), (vec![7, 70], 1)], + ); + rows.insert( + h, + vec![(vec![2, 20], 1), (vec![3, 21], 1), (vec![9, 99], 1)], + ); + let want = run_once(atoms, false, &rows); + let got = run_once(atoms, true, &rows); + assert_eq!(got, want, "STAR-3 acyclic WCOJ must match binary"); + } + + // --- Shape 1 with SELF-JOIN: F(v0,v1,v2), G(v0,v3), G(v1,v4) -------- + { + let atoms: &[(FunctionId, &[u32])] = &[(f, &[0, 1, 2]), (g, &[0, 3]), (g, &[1, 4])]; + assert!( + detect_cyclic_cq(&plan_of(atoms), true).is_some(), + "self-join star fires WCOJ" + ); + let mut rows = HashMap::new(); + rows.insert( + f, + vec![ + (vec![1, 2, 100], 1), + (vec![1, 3, 101], 1), + (vec![4, 2, 102], 1), + ], + ); + rows.insert( + g, + vec![ + (vec![1, 10], 1), + (vec![1, 11], 1), + (vec![2, 20], 1), + (vec![3, 21], 1), + (vec![4, 40], 1), + (vec![9, 99], 1), + ], + ); + let want = run_once(atoms, false, &rows); + let got = run_once(atoms, true, &rows); + assert_eq!(got, want, "STAR-3 self-join acyclic WCOJ must match binary"); + } + + // --- Shape 2: 4-atom STAR (3-way self-join) ------------------------- + // F(v0,v1,v2,v3), G(v0,v4), G(v1,v5), G(v2,v6) + { + let atoms: &[(FunctionId, &[u32])] = + &[(f, &[0, 1, 2, 3]), (g, &[0, 4]), (g, &[1, 5]), (g, &[2, 6])]; + assert!( + detect_cyclic_cq(&plan_of(atoms), true).is_some(), + "4-star fires WCOJ" + ); + let mut rows = HashMap::new(); + rows.insert(f, vec![(vec![1, 2, 3, 100], 1), (vec![1, 2, 8, 101], 1)]); + rows.insert( + g, + vec![ + (vec![1, 10], 1), + (vec![1, 11], 1), + (vec![2, 20], 1), + (vec![3, 30], 1), + (vec![8, 80], 1), + (vec![9, 99], 1), + ], + ); + let want = run_once(atoms, false, &rows); + let got = run_once(atoms, true, &rows); + assert_eq!(got, want, "STAR-4 acyclic WCOJ must match binary"); + } + + // --- Shape 3: 3-atom PATH R(a,b), S(b,c), T(c,d) ------------------- + { + let r = FunctionId::new(3); + let s = FunctionId::new(4); + let t = FunctionId::new(5); + let atoms: &[(FunctionId, &[u32])] = &[(r, &[0, 1]), (s, &[1, 2]), (t, &[2, 3])]; + assert!( + detect_cyclic_cq(&plan_of(atoms), true).is_some(), + "path fires WCOJ" + ); + let mut rows = HashMap::new(); + rows.insert(r, vec![(vec![1, 2], 1), (vec![1, 5], 1), (vec![9, 2], 1)]); + rows.insert(s, vec![(vec![2, 3], 1), (vec![5, 6], 1), (vec![2, 7], 1)]); + rows.insert(t, vec![(vec![3, 4], 1), (vec![6, 8], 1), (vec![7, 100], 1)]); + let want = run_once(atoms, false, &rows); + let got = run_once(atoms, true, &rows); + assert_eq!(got, want, "PATH-3 acyclic WCOJ must match binary"); + } + } + + /// The general WCOJ detector: fires on cyclic >=3-atom bodies, falls back to + /// the binary chain (returns `None`) on 2-atom / acyclic / const shapes. + #[test] + fn detect_cyclic_cq_boundary() { + let r = FunctionId::new(0); + let s = FunctionId::new(1); + + // 2 atoms: never WCOJ (acyclic, no blowup). + assert!(detect_cyclic_cq(&plan_of(&[(r, &[0, 1]), (s, &[1, 2])]), false).is_none()); + + // 3-atom ACYCLIC (a path a-b-c-d): stays on the binary chain. + let path = plan_of(&[(r, &[0, 1]), (r, &[1, 2]), (r, &[2, 3])]); + assert!(!is_cyclic_cq(&[vec![0, 1], vec![1, 2], vec![2, 3]])); + assert!(detect_cyclic_cq(&path, false).is_none()); + + // 3-atom triangle (cyclic): fires the general path, with one delta query + // per atom and every join var bound. + let tri = plan_of(&[(r, &[0, 1]), (r, &[1, 2]), (r, &[2, 0])]); + assert!(is_cyclic_cq(&[vec![0, 1], vec![1, 2], vec![2, 0]])); + let cq = detect_cyclic_cq(&tri, false).expect("triangle is a cyclic CQ"); + assert_eq!(cq.drivers.len(), 3, "one delta query per atom"); + + // 4-clique-with-payload (term-encoding shape: each edge atom carries a + // fresh eclass + extra payload column). Cyclic core a,b,c,d; fires. + let clique = plan_of(&[ + (r, &[0, 1, 10, 20]), + (r, &[0, 2, 11, 21]), + (r, &[0, 3, 12, 22]), + (r, &[1, 2, 13, 23]), + (r, &[1, 3, 14, 24]), + (r, &[2, 3, 15, 25]), + ]); + let cqc = detect_cyclic_cq(&clique, false).expect("4-clique is a cyclic CQ"); + assert_eq!(cqc.drivers.len(), 6); + // Each driver recovers every NON-driver atom's payload (n_atoms-1). + for d in &cqc.drivers { + assert_eq!(d.recovers.len(), 5); + } + } + + fn delta( + func: FunctionId, + rows: &[(&[u32], isize)], + ) -> HashMap, isize)>> { + let mut m = HashMap::new(); + m.insert(func, rows.iter().map(|(r, w)| (r.to_vec(), *w)).collect()); + m + } + + /// CRUX #1 + #2: build-once + drive-epochs feeding ONLY deltas, and the join + /// stays incremental across epochs (epoch 2 fed only the new edge emits ONLY + /// the new hop, not a re-derivation). + #[test] + fn dd_native_join_is_incremental() { + let f = FunctionId::new(0); + let plan = tc_plan(f); + let mut pj = PersistentDdJoin::build(&plan).expect("build"); + + // Epoch 1: seed edges (1,2),(2,3). Only hop is (1,2,3). + let out1 = pj + .step(&delta(f, &[(&[1, 2], 1), (&[2, 3], 1)])) + .expect("step1"); + assert_eq!(out1, vec![(vec![1, 2, 3], 1)], "first hop"); + + // Epoch 2: add ONLY new edge (3,4). Incremental join must emit ONLY the + // new binding (2,3,4) — NOT re-derive (1,2,3). + let out2 = pj.step(&delta(f, &[(&[3, 4], 1)])).expect("step2"); + assert_eq!(out2, vec![(vec![2, 3, 4], 1)], "only the new hop"); + + // CRUX #3: retract edge (2,3). Two bindings used it; both retract + // (negative weight) — bit-exact retraction via DD signed weights. + let mut out3 = pj.step(&delta(f, &[(&[2, 3], -1)])).expect("step3"); + out3.sort(); + assert_eq!( + out3, + vec![(vec![1, 2, 3], -1), (vec![2, 3, 4], -1)], + "retraction propagates" + ); + } +} diff --git a/egglog-experimental/flowlog/src/engine.rs b/egglog-experimental/flowlog/src/engine.rs new file mode 100644 index 0000000..18cada5 --- /dev/null +++ b/egglog-experimental/flowlog/src/engine.rs @@ -0,0 +1,89 @@ +//! Thin wrapper around the build-time-generated flowlog-rs +//! `DatalogIncrementalEngine`. +//! +//! The generated module (from `build.rs` compiling `transitive_step.dl` in +//! `DatalogInc` mode) is `include!`d here so the generated symbols stay +//! confined to this file. Per SPIKE_RESULTS.md (confirmed empirically) the +//! generated API is: +//! +//! - `DatalogIncrementalEngine::new(workers) -> Self` (spawns timely workers), +//! - `engine.begin()` (auto-called by the first `insert_*`/`remove_*`), +//! - `engine.insert_(Vec)` / `remove_(Vec)` +//! (relation names lowercased: `insert_edge`, `insert_path`), +//! - `engine.commit() -> IncrementalResults` (steps ONE epoch to fixpoint, +//! blocks, returns this epoch's per-output deltas), +//! - `rel::Edge` / `rel::Path` / `rel::Hop` are **tuple aliases** `(i32, i32)`, +//! - `IncrementalResults.hop: Vec<(rel::Hop, i32)>` (field = lowercase rel name; +//! the `i32` is the differential-dataflow multiplicity diff). +//! +//! Because the `.dl` join `hop(x,z) :- path(x,y), edge(y,z)` is **non-recursive**, +//! one `commit()` is exactly one round of the join over the staged delta — which +//! is what makes one egglog `run_rules` call = one bounded hop. + +#[allow(clippy::all)] +#[allow(dead_code)] +#[allow(unused)] +mod gen { + // The generated file is fully self-contained (it pulls the + // flowlog-runtime re-exports of timely / differential-dataflow itself). + include!(concat!(env!("OUT_DIR"), "/transitive_step.rs")); +} + +use gen::DatalogIncrementalEngine; + +/// Owns the live flowlog incremental engine plus the host-side per-round +/// feedback buffer (`pending_path`): the `path` rows derived in the last +/// committed hop, to be staged as the next round's `insert_path` delta. +pub struct FlowEngine { + engine: DatalogIncrementalEngine, + /// `path` rows derived last round, awaiting feed-back next round. + pending_path: Vec<(i32, i32)>, +} + +impl FlowEngine { + /// Spawn a fresh single-worker incremental engine. + pub fn new() -> Self { + FlowEngine { + engine: DatalogIncrementalEngine::new(1), + pending_path: Vec::new(), + } + } + + /// Stage `edge(src, dst)` insert deltas (auto-begins the txn). + pub fn insert_edge(&mut self, rows: &[(i32, i32)]) { + if rows.is_empty() { + return; + } + self.engine.insert_edge(rows.to_vec()); + } + + /// Stage `path(src, dst)` insert deltas (auto-begins the txn). + pub fn insert_path(&mut self, rows: &[(i32, i32)]) { + if rows.is_empty() { + return; + } + self.engine.insert_path(rows.to_vec()); + } + + /// Step exactly one epoch to fixpoint and return this epoch's `hop` deltas + /// as `(x, z, diff)`. The non-recursive join means these are exactly the + /// new one-hop extensions caused by the rows staged since the last commit. + pub fn commit_hop(&mut self) -> Vec<(i32, i32, i32)> { + let results = self.engine.commit(); + results + .hop + .into_iter() + .map(|(t, d): ((i32, i32), i32)| (t.0, t.1, d)) + .collect() + } + + /// Record the `path` rows to feed back next round. + pub fn set_pending_path(&mut self, rows: Vec<(i32, i32)>) { + self.pending_path = rows; + } + + /// Take (and clear) the pending feed-back `path` rows. + pub fn take_pending_path(&mut self) -> Vec<(i32, i32)> { + std::mem::take(&mut self.pending_path) + } +} diff --git a/egglog-experimental/flowlog/src/external_func.rs b/egglog-experimental/flowlog/src/external_func.rs new file mode 100644 index 0000000..bfc512e --- /dev/null +++ b/egglog-experimental/flowlog/src/external_func.rs @@ -0,0 +1,99 @@ +//! External-function (primitive) registry for the FlowLog backend. +//! +//! ## Milestone 1 scope +//! +//! M1's proof program (a non-recursive transitive-closure step) uses NO +//! primitives. This module exists so the [`Backend`](egglog_backend_trait::Backend) +//! surface is complete (`register_external_func` / `free_external_func` / +//! `new_panic`) and so later milestones can wire primitive evaluation through +//! the embedded `Database` the way the Feldera backend does. It is a thin +//! side-table indexed by the same [`ExternalFunctionId`] the `Database` +//! assigns, tracking names + panic sentinels. Mirrors the Feldera backend. + +use egglog_backend_trait::{ExecutionState, ExternalFunction, ExternalFunctionId, Value}; +use egglog_numeric_id::NumericId; + +/// Either a real registered primitive, a panic sentinel, or a freed slot. +enum Slot { + Func, + Panic(#[allow(dead_code)] String), + Free, +} + +/// A side-table of external-function metadata, indexed by the +/// [`ExternalFunctionId`] the embedded `Database` assigned. `add_*_at` keeps +/// this `Vec` aligned with the Database's id allocation (ids advance in +/// lockstep). +#[derive(Default)] +pub struct ExternalFuncRegistry { + slots: Vec, + names: Vec>, +} + +impl ExternalFuncRegistry { + fn ensure_len(&mut self, idx: usize) { + while self.slots.len() <= idx { + self.slots.push(Slot::Free); + self.names.push(None); + } + } + + /// Record a primitive at the id the Database assigned. + pub fn add_func_at( + &mut self, + id: ExternalFunctionId, + _func: Box, + ) { + let idx = id.rep() as usize; + self.ensure_len(idx); + self.slots[idx] = Slot::Func; + } + + /// Record a panic sentinel at the id the Database assigned. + pub fn add_panic_at(&mut self, id: ExternalFunctionId, message: String) { + let idx = id.rep() as usize; + self.ensure_len(idx); + self.slots[idx] = Slot::Panic(message); + } + + /// The display name of a primitive, if recorded. + #[allow(dead_code)] + pub fn name(&self, id: ExternalFunctionId) -> Option<&str> { + self.names.get(id.rep() as usize).and_then(|n| n.as_deref()) + } + + /// Tombstone a slot. Idempotent. + pub fn free(&mut self, id: ExternalFunctionId) { + if let Some(slot) = self.slots.get_mut(id.rep() as usize) { + *slot = Slot::Free; + } + } + + /// The panic message for a deferred-panic id, if any. + #[allow(dead_code)] + pub fn panic_message(&self, id: ExternalFunctionId) -> Option<&str> { + match self.slots.get(id.rep() as usize) { + Some(Slot::Panic(m)) => Some(m.as_str()), + _ => None, + } + } +} + +/// A real, invokable panic sentinel registered into the Database by +/// `Backend::new_panic`. Invoking it panics with the recorded message. +#[derive(Clone)] +pub struct PanicFunc { + message: String, +} + +impl PanicFunc { + pub fn new(message: String) -> Self { + PanicFunc { message } + } +} + +impl ExternalFunction for PanicFunc { + fn invoke(&self, _state: &mut ExecutionState, _args: &[Value]) -> Option { + panic!("{}", self.message); + } +} diff --git a/egglog-experimental/flowlog/src/interpret.rs b/egglog-experimental/flowlog/src/interpret.rs new file mode 100644 index 0000000..b61d062 --- /dev/null +++ b/egglog-experimental/flowlog/src/interpret.rs @@ -0,0 +1,681 @@ +//! Host-side iteration driver for the FlowLog backend. +//! +//! One `run_rules` call = **one bounded egglog iteration**. The body join runs +//! on the in-process, build-once, epoch-driven raw differential-dataflow +//! dataflow (`crate::dd_native`); this module owns the orchestration around it: +//! +//! 1. snapshot the relation mirror (the read view for this iteration — all rules +//! match against the same pre-iteration state, egglog's semi-naive "match the +//! old database, then apply" model for a single hop); +//! 2. for each rule, drive its persistent DD join with the per-relation signed +//! delta vs. what that join was last fed, then re-run any body primitives +//! (`!=` guards, value-computing prims) host-side over the produced bindings; +//! 3. for every surviving binding, execute the head ops in order — `set` / +//! `remove` / `subsume` writes, RHS `lookup` (eq-sort constructor: create on +//! miss), RHS primitive `call`, `union`, `panic`; +//! 4. apply all collected writes/removes to the mirror and resolve +//! functional-dependency conflicts per each touched function's merge mode. +//! +//! ## The engine split (mirrors Feldera Stage C) +//! +//! The relational table-atom join is the ONLY thing on the engine, and it is the +//! ONLY join path — there is no host nested-loop fallback. Any rule the DD plan +//! cannot lower (a binding row exceeding the fixed width cap [`dd_native::W`], or +//! any shape `plan_join` rejects) `panic!`s with a specific reason. The primitive +//! tail + head actions are applied HOST-side here, exactly mirroring Feldera's +//! "table join on the engine, prim tail host-side" split. +//! +//! Primitives are invoked through `Database::with_execution_state`, so they see +//! the same interned base `Value`s the frontend created — giving the FlowLog +//! backend bit-for-bit value parity with the reference backend (no `eval_prim` +//! trait change — flowlog keeps its zero-trait-change posture). + +use anyhow::{anyhow, Result}; +use egglog_backend_trait::{FunctionId, Value}; +use egglog_numeric_id::NumericId; +use hashbrown::{HashMap, HashSet}; + +use crate::compile::{row_col, slot_lookup, BodyOp, HeadOp, Row, RuleIr, Slot}; +use crate::EGraph; + +/// Binding environment: variable id → bound `u32` value. +pub(crate) type Env = HashMap; + +/// A pending write to apply after all matches are computed. +enum Write { + /// Insert/overwrite a full row. + Set(FunctionId, Row), + /// Retract by key (the slots address inputs for a function, whole row for a + /// relation). + Remove(FunctionId, Vec), +} + +/// One bounded egglog iteration with the body join running on the in-process, +/// build-once, epoch-driven raw differential-dataflow dataflow +/// (`crate::dd_native`). This is the FlowLog analog of the Feldera backend's +/// `interpret::run_iteration` + `persistent_bindings`. +/// +/// Per rule: compute the signed `+/-` delta of each body relation vs the rows +/// last fed to that rule's persistent DD join, `step` the join (which feeds ONLY +/// the delta into never-cleared InputSessions — genuinely incremental), turn the +/// positive binding deltas into envs, re-run body prims host-side (value prims / +/// guards the engine keeps off-circuit), then apply head actions. Writes + +/// FD-merge are applied so results are bit-exact. +pub fn run_iteration(eg: &mut EGraph, rule_idxs: &[usize]) -> Result { + // FlowLog runs the fast RELATIONAL term-encoding: EVERY rule — including the + // `@rebuild_rule*` canonicalization rules — lowers as an ordinary rule and + // joins on the fused DD worker (the sound `step_symmetric` path: `view ⋈ @UF_Sf` + // + full-recanon action). No native-UF host-pass, no synthetic `@DispΔ` + // relation, no rebuild-rule recognizer, no δUF-scoping special-casing. + + // Snapshot the read view: rules match against the pre-iteration mirror. + // + // The snapshot shares each function's row set by `Rc` rather than + // deep-cloning every row: this is O(#functions), not O(state). Mutations to + // the mirror this call (head writes, hash-cons in `lookup_or_create`, merge + // resolution) go through `Rc::make_mut`, which copy-on-writes only the + // functions actually changed while this snapshot is alive — so `read` keeps + // the start-of-call contents and rules all match the pre-iteration state. + let read: HashMap>> = eg + .mirror + .iter() + .map(|(f, set)| (*f, std::rc::Rc::clone(set))) + .collect(); + + // Snapshot the fresh-id counter: any hash-cons (`lookup_or_create`) this + // call advances it, the O(1) signal that a new term row was created. + let next_id_at_start = eg.next_id; + + let mut writes: Vec = Vec::new(); + let mut touched: HashSet = HashSet::new(); + // Iteration-scoped `key -> output` index for `lookup_or_create` (eq-sort + // constructor hash-cons). Built lazily per function so repeated lookups in + // one iteration are O(1) instead of rescanning the growing mirror each time. + let mut lookup_index: HashMap, u32>> = HashMap::new(); + + // Collect each rule's index + IR up front (clone to avoid borrow conflicts + // while we also mutate the mirror via lookups). EVERY rule — including the + // `@rebuild_rule*` canonicalization rules — takes the ordinary atom-rule path + // (no recognizer, no rewrite, no drop): they are relational rules + // (`view ⋈ @UF_Sf` body + full-recanon head) and join on the fused DD worker + // like any other rule. + let rules: Vec<(usize, RuleIr)> = rule_idxs + .iter() + .filter_map(|&i| eg.rules.get(i).and_then(|r| r.clone()).map(|r| (i, r))) + .collect(); + + // Compute every rule's binding envs FIRST (so the whole atom-bearing ruleset + // runs on ONE fused DD worker in a single epoch — `fused_bindings`), THEN + // apply head actions in the original rule firing order. Atom-less rules + // (`(rule () …)`) have no input relation to drive the DD dataflow, so they + // stay host-side (fire once); they are computed inline below. + let envs_by_rule = fused_bindings(eg, &read, &rules)?; + + for ((idx, rule), envs) in rules.iter().zip(envs_by_rule.into_iter()) { + let _ = idx; + for mut env in envs { + apply_head( + eg, + &rule.head, + &mut env, + &mut writes, + &mut touched, + &mut lookup_index, + )?; + } + } + + // Apply collected writes to the mirror. + // + // Removes are BATCHED per function: applying each `Write::Remove` with its + // own `set.retain` scan is O(|removes| · |state|) — quadratic. We collect all + // retraction keys per function into a hash set, then do a SINGLE `retain` + // pass per touched function: O(|state|) total. Removes are applied FIRST + // (batched), then Sets — preserving the term encoder's `(@uf)` "delete old + // leader, set new leader" delete-then-set ordering. + // + // `changed` is computed INCREMENTALLY as writes land (O(delta)), not via a + // full before/after content compare. A hash-cons in `lookup_or_create` + // always allocates a fresh id, so any term created this call advances + // `next_id` — that alone is a real mirror change. + let mut changed = eg.next_id != next_id_at_start; + let mut removes_by_func: HashMap>)> = HashMap::new(); + let mut sets: Vec<(FunctionId, Row)> = Vec::new(); + for w in writes { + match w { + Write::Set(f, row) => sets.push((f, row)), + Write::Remove(f, key) => { + let entry = removes_by_func + .entry(f) + .or_insert_with(|| (key.len(), HashSet::new())); + entry.1.insert(key.into_boxed_slice()); + } + } + } + for (f, (keylen, keys)) in removes_by_func { + if let Some(set) = eg.mirror.get_mut(&f) { + let before_len = set.len(); + std::rc::Rc::make_mut(set).retain(|row| { + let k: Box<[u32]> = (0..keylen).map(|i| row_col(row, i)).collect(); + !keys.contains(&k) + }); + changed |= set.len() != before_len; + } + } + let mut touched_keys: HashMap>> = HashMap::new(); + for (f, row) in sets { + let inputs_len = eg.info(f).arity.saturating_sub(1); + let key: Vec = (0..inputs_len).map(|i| row_col(&row, i)).collect(); + let inserted = std::rc::Rc::make_mut(eg.mirror.entry(f).or_default()).insert(row.clone()); + changed |= inserted; + touched_keys.entry(f).or_default().insert(key); + } + + // Resolve FD conflicts on every function a head action wrote to (a `set` + // can introduce two rows sharing a key that must merge per the merge mode). + let empty_keys: HashSet> = HashSet::new(); + for &f in &touched { + let keys = touched_keys.get(&f).unwrap_or(&empty_keys); + changed |= eg.resolve_merge(f, keys); + } + + Ok(changed) +} + +/// Derive a stable ruleset LABEL for a `run_rules` call from the rules it runs, +/// for `FLOWLOG_DD_RULESET_PROF`. The backend trait does not carry the ruleset +/// name down to `run_rules`, but the term encoder gives each maintenance rule a +/// `fresh()`-suffixed name with a stable PREFIX that identifies its ruleset +/// (`uf_update*` / `singleparent*` / `uf_function_index*` / `congruence_rule*` / +/// `rebuild_rule*` / `merge_rule*` / `merge_cleanup*` / `delete_rule*`); user +/// rewrite rules are named by their full s-expression text. We map each rule +/// name to its category, then label the call by the categories present (all +/// rules in one `run_rules` call belong to one egglog ruleset). +/// Map a single rule NAME to its bucket label. Maintenance rules emitted by the +/// term encoder (proof_encoding.rs) carry a stable, `fresh()`-suffixed name; +/// most are `@`-prefixed (`@uf_update`, `@congruence_rule`, …) but +/// `singleparent@uf_update` is not. Match the identifying substring so the +/// leading `@` and trailing digits don't matter. Order: most specific first +/// (`singleparent` and the `_subsume` variant before their bare forms; +/// `uf_function_index` before `uf_update`). +pub(crate) fn rule_category(name: &str) -> &'static str { + const MAINT: &[(&str, &str)] = &[ + ("singleparent", "single_parent"), + ("uf_function_index", "uf_function_index"), + // PR #782 native-UF drain (the `@uf_change_drain` ruleset). Must be + // matched BEFORE `uf_update` (substring overlap is none, but keep the + // most-specific UF rules grouped). Dropped under `--native-uf`. + ("uf_change_drain", "uf_change_drain"), + ("uf_update", "path_compress/uf_update"), + ("delete_rule_subsume", "delete_subsume"), + ("delete_rule", "delete_subsume"), + // `@congruence_rule` and `@rebuild_rule` are both in the term encoder's + // `rebuilding` ruleset and run in ONE fused `run_rules` call; split them + // into distinct buckets so the native-UF-addressable cost + // (canonicalization, which joins function rows against `@uf`) can be + // separated from congruence detection (which stays relational under a + // native UF). + ("congruence_rule", "congruence"), + ("rebuild_rule", "canonicalize"), + ("merge_cleanup", "rebuilding_cleanup"), + ("merge_rule", "merge_rule"), + ]; + for (needle, label) in MAINT { + if name.contains(needle) { + return label; + } + } + if name.starts_with("eval_actions") { + return "eval_actions"; + } + "" +} + +/// Compute every rule's binding envs in ONE fused pass: the whole atom-bearing +/// ruleset's body joins run on a SINGLE shared timely worker +/// ([`dd_native::FusedDdJoin`]) clocked once this iteration, then each rule's +/// host-side prim tail is re-run over its own bindings. Atom-less rules +/// (`(rule () …)`) have no input relation to drive the DD dataflow, so they are +/// fired once host-side. Returns a `Vec>` parallel to `rules` (same +/// order), ready for `apply_head`. +fn fused_bindings( + eg: &mut EGraph, + read: &HashMap>>, + rules: &[(usize, RuleIr)], +) -> Result>> { + use crate::dd_native; + + let prof = dd_native::prof_enabled(); + let rs_prof = dd_native::ruleset_prof_enabled(); + // Per-ruleset attribution: the wall clock for the WHOLE atom-bearing path + // this call, plus the per-bucket nanos we accumulate below. The call's time + // is later apportioned across the rule CATEGORIES present (so `@rebuild_rule` + // and `@congruence_rule`, fused into one timely step, land in distinct + // `canonicalize`/`congruence` buckets). + let rs_t_total = std::time::Instant::now(); + let mut rs_delta_ns: u64 = 0; + let mut rs_prim_ns: u64 = 0; + let mut rs_feed_ns: u64 = 0; + let mut rs_step_ns: u64 = 0; + let mut rs_delta_rows: u64 = 0; + let mut out: Vec> = vec![Vec::new(); rules.len()]; + + // Partition: atom-bearing rules drive the fused DD worker; atom-less rules + // fire once host-side. Record each atom-bearing rule's POSITION in `rules` so + // we can scatter the fused output back into `out` in the caller's order. + let mut atom_positions: Vec = Vec::new(); + let mut atom_rule_idxs: Vec = Vec::new(); + for (pos, (idx, rule)) in rules.iter().enumerate() { + let _ = idx; + let has_atoms = rule.body.iter().any(|op| matches!(op, BodyOp::Atom(_))); + if has_atoms { + atom_positions.push(pos); + atom_rule_idxs.push(*idx); + } else { + // Atom-less rule: fire once (presence in `seen` = already fired). + if eg.seen.contains_key(idx) { + continue; + } + eg.seen.insert(*idx, ()); + let mut envs: Vec = vec![Env::new()]; + for op in &rule.body { + envs = step_prim(eg, op, envs)?; + if envs.is_empty() { + break; + } + } + out[pos] = envs; + } + } + + if atom_positions.is_empty() { + return Ok(out); + } + + // The fused join is keyed by the SORTED atom-bearing rule-index list (the + // ruleset identity), exactly like feldera's `FusedJoin`. Build it ONCE + // (lazily) per distinct ruleset, planning each rule. Any shape `plan_join` + // rejects PANICS (no host fallback; the DD dataflow is the only join path). + let mut key: Vec = atom_rule_idxs.clone(); + key.sort_unstable(); + + if !eg.dd_fused.contains_key(&key) { + // Plan in the SAME order as `atom_positions` so the fused build order + // matches our scatter order (the fused join preserves plan order). + let mut plans: Vec<(usize, dd_native::JoinPlan)> = Vec::with_capacity(atom_positions.len()); + for (&pos, &idx) in atom_positions.iter().zip(atom_rule_idxs.iter()) { + let rule = &rules[pos].1; + let plan = match dd_native::plan_join(rule) { + Ok(p) => p, + Err(reason) => panic!( + "FlowLog DD join cannot lower rule {:?}: {reason} \ + (no host fallback; the DD dataflow is the only join path)", + rule.name + ), + }; + plans.push((idx, plan)); + } + // `--wcoj`: enable the worst-case-optimal triangle delta query. The + // build detects the triangle shape per rule; non-triangle rules in the + // ruleset keep the binary `.join` chain (hybrid). Off ⇒ byte-identical + // to the pre-WCOJ build. EVERY rule — including the `@rebuild_rule*` + // canonicalization rules — runs through the same general fused join (the + // sound `step_symmetric` path); there is no δUF-scoping fast-rebuild + // variant anymore. + let wcoj = eg.wcoj_enabled; + let fused = dd_native::FusedDdJoin::build(&plans, wcoj, false)?; + eg.dd_fused.insert(key.clone(), fused); + } + + // The fused join's internal rule order (its build order = `atom_positions` + // order). Map each fused output slot back to the caller `rules` position and + // capture each rule's canonical var order. + let (fused_rule_idxs, fused_body_funcs): (Vec, Vec>) = { + let fused = eg.dd_fused.get(&key).expect("fused join present"); + ( + fused.rule_indices(), + (0..fused.rule_indices().len()) + .map(|p| fused.rule_body_funcs(p).to_vec()) + .collect(), + ) + }; + // The fused build order equals `atom_rule_idxs` (we built it that way), so + // map fused position -> caller `rules` position via `atom_positions`. + debug_assert_eq!(fused_rule_idxs, atom_rule_idxs, "fused build order"); + + // Each atom-bearing rule's canonical var order (for env reconstruction). + let var_orders: Vec> = atom_positions + .iter() + .map(|&pos| { + let rule = &rules[pos].1; + dd_native::plan_join(rule) + .expect("plan re-derivable") + .var_order() + }) + .collect(); + + // Distinct relations across the whole ruleset → ONE combined signed delta map + // fed into the fused worker's SHARED inputs. The `fed` snapshot is per-ruleset + // (the fused join's identity), diffed against the live mirror like the per-rule + // `dd_native_fed`. + let t_delta = std::time::Instant::now(); + let empty_set: std::rc::Rc> = std::rc::Rc::new(HashSet::new()); + let mut all_funcs: Vec = Vec::new(); + for bf in &fused_body_funcs { + for &f in bf { + if !all_funcs.contains(&f) { + all_funcs.push(f); + } + } + } + let mut delta: HashMap, isize)>> = HashMap::new(); + { + let fed = eg.dd_fused_fed.entry(key.clone()).or_default(); + for &f in &all_funcs { + let cur = read.get(&f).cloned().unwrap_or_else(|| empty_set.clone()); + let prev = fed.entry(f).or_insert_with(|| empty_set.clone()); + if std::rc::Rc::ptr_eq(&cur, prev) { + *prev = cur; + continue; + } + let mut rows: Vec<(Vec, isize)> = Vec::new(); + for r in cur.iter() { + if !prev.contains(r) { + rows.push((r.to_vec(), 1)); + } + } + for r in prev.iter() { + if !cur.contains(r) { + rows.push((r.to_vec(), -1)); + } + } + if !rows.is_empty() { + rs_delta_rows += rows.len() as u64; + delta.insert(f, rows); + } + *prev = cur; + } + } + let delta_elapsed = t_delta.elapsed().as_nanos() as u64; + if prof { + dd_native::PROF_DELTA_NS.fetch_add(delta_elapsed, std::sync::atomic::Ordering::Relaxed); + } + rs_delta_ns += delta_elapsed; + + // ONE step of the shared worker for the WHOLE ruleset. `step` updates the + // global PROF_FEED_NS / PROF_STEP_NS counters when profiling is on (which it + // is whenever either env var is set), so snapshot them before/after to split + // this ruleset's feed vs worker_step time. + use std::sync::atomic::Ordering as ProfOrd; + let feed_before = dd_native::PROF_FEED_NS.load(ProfOrd::Relaxed); + let step_before = dd_native::PROF_STEP_NS.load(ProfOrd::Relaxed); + eg.dd_rule_runs += atom_positions.len() as u64; + let per_rule_bindings = { + let fused = eg.dd_fused.get_mut(&key).expect("fused join present"); + fused.step(&delta)? + }; + if rs_prof { + rs_feed_ns += dd_native::PROF_FEED_NS + .load(ProfOrd::Relaxed) + .wrapping_sub(feed_before); + rs_step_ns += dd_native::PROF_STEP_NS + .load(ProfOrd::Relaxed) + .wrapping_sub(step_before); + } + + // Per-rule positive-binding count = the fused join's workload proxy for each + // rule (used below to apportion the single fused worker_step across the rule + // CATEGORIES present in this call). Length is parallel to `atom_positions`. + let mut rs_pos_bindings: Vec = vec![0; atom_positions.len()]; + + // Turn each rule's positive binding deltas into envs; re-run its body prims + // host-side. Negative weights are integral bookkeeping (a body row retracted) + // — egglog heads are monotone-fire, so we do NOT re-fire on disappearance. + let t_prim = std::time::Instant::now(); + for (fpos, bindings) in per_rule_bindings.into_iter().enumerate() { + let caller_pos = atom_positions[fpos]; + let rule = &rules[caller_pos].1; + let var_order = &var_orders[fpos]; + let mut envs: Vec = Vec::new(); + for (bind, w) in &bindings { + if *w <= 0 { + continue; + } + rs_pos_bindings[fpos] += 1; + let mut env: Env = Env::new(); + for (i, &v) in var_order.iter().enumerate() { + env.insert(v, bind[i]); + } + let mut es: Vec = vec![env]; + for op in &rule.body { + if let BodyOp::Prim { .. } = op { + es = step_prim(eg, op, es)?; + } + } + envs.extend(es); + } + out[caller_pos] = envs; + } + + let prim_elapsed = t_prim.elapsed().as_nanos() as u64; + if prof { + dd_native::PROF_PRIM_NS.fetch_add(prim_elapsed, std::sync::atomic::Ordering::Relaxed); + } + rs_prim_ns += prim_elapsed; + + if rs_prof { + let rs_total_ns = rs_t_total.elapsed().as_nanos() as u64; + // Several rule CATEGORIES (e.g. `canonicalize` = `@rebuild_rule` and + // `congruence` = `@congruence_rule`) run in ONE fused timely step, so + // there is no per-category wall clock. Apportion this call's measured + // buckets across the categories present, weighted by each category's + // share of POSITIVE output bindings (the join workload it produced). If + // no rule produced output this call, fall back to an even split by rule + // count so the call's wall time is still attributed. + let mut cat_w: HashMap<&'static str, u64> = HashMap::new(); + let mut cat_n: HashMap<&'static str, u64> = HashMap::new(); + // Cross-check: nanos of (apportioned) worker_step attributable to rules + // whose BODY reads a `@uf` table (`UF_*` relation) — the native-UF- + // addressable fraction proxy. + let mut uf_body_w: u64 = 0; + for (fpos, &pos) in atom_positions.iter().enumerate() { + let rule = &rules[pos].1; + let cat = rule_category(&rule.name); + let w = rs_pos_bindings[fpos]; + *cat_w.entry(cat).or_default() += w; + *cat_n.entry(cat).or_default() += 1; + let reads_uf = rule.body.iter().any( + |op| matches!(op, BodyOp::Atom(a) if eg.relation_name(a.func).contains("UF_")), + ); + if reads_uf { + uf_body_w += w; + } + } + let total_w: u64 = cat_w.values().sum(); + let total_n: u64 = cat_n.values().sum(); + // Worker_step nanos apportioned to UF-body-reading rules. + let uf_step_ns = if total_w > 0 { + (rs_step_ns as u128 * uf_body_w as u128 / total_w as u128) as u64 + } else { + 0 + }; + dd_native::ruleset_uf_body_record(uf_step_ns, rs_step_ns); + for (cat, &w) in &cat_w { + let n = cat_n[cat]; + // share by binding workload, else by rule count. + let (num, den): (u128, u128) = if total_w > 0 { + (w as u128, total_w as u128) + } else { + (n as u128, total_n.max(1) as u128) + }; + let part = |v: u64| (v as u128 * num / den) as u64; + dd_native::ruleset_prof_record( + cat, + part(rs_total_ns), + part(rs_step_ns), + part(rs_feed_ns), + part(rs_prim_ns), + part(rs_delta_ns), + part(rs_delta_rows), + ); + } + } + + Ok(out) +} + +/// Evaluate a primitive body op over each binding env, returning the new list of +/// envs. A value-computing prim binds (or checks) its return var; a guard prim +/// (`!=`) that fails prunes the env. Table atoms are NOT handled here — they run +/// on the DD dataflow; this is only the host-side primitive tail. +pub(crate) fn step_prim(eg: &mut EGraph, op: &BodyOp, envs: Vec) -> Result> { + let BodyOp::Prim { id, args, ret } = op else { + unreachable!("step_prim called on a non-primitive body op"); + }; + let mut out = Vec::new(); + for env in envs { + let resolved: Option> = args + .iter() + .map(|s| slot_lookup(s, &|v| env.get(&v).copied()).map(Value::new)) + .collect(); + let Some(argv) = resolved else { continue }; + let result = eg.eval_prim_internal(*id, &argv); + let Some(result) = result else { + // Primitive failed (e.g. `!=` of equal args) — prune. + continue; + }; + match ret { + Slot::Var(v) => { + let mut next = env.clone(); + match next.get(v) { + Some(&existing) if existing != result.rep() => continue, + _ => { + next.insert(*v, result.rep()); + } + } + out.push(next); + } + Slot::Const(c) => { + if *c == result.rep() { + out.push(env); + } + } + } + } + Ok(out) +} + +/// Execute the head ops for one binding, accumulating writes. +fn apply_head( + eg: &mut EGraph, + head: &[HeadOp], + env: &mut Env, + writes: &mut Vec, + touched: &mut HashSet, + lookup_index: &mut HashMap, u32>>, +) -> Result<()> { + for op in head { + match op { + HeadOp::Set { func, slots } => { + let row = build_row(slots, env)?; + touched.insert(*func); + writes.push(Write::Set(*func, row)); + } + HeadOp::Remove { func, slots } => { + let key: Vec = slots + .iter() + .map(|s| resolve(s, env)) + .collect::>()?; + touched.insert(*func); + writes.push(Write::Remove(*func, key)); + } + HeadOp::Subsume { func, .. } => { + // Subsumption is not tracked; treat as a no-op (the row stays + // present). `supports_subsumption()` is false, so the frontend + // never relies on subsumed-row filtering on this backend. + touched.insert(*func); + } + HeadOp::Lookup { func, args, ret } => { + let key: Vec = args + .iter() + .map(|s| resolve(s, env).map(Value::new)) + .collect::>()?; + let val = lookup_or_create(eg, *func, &key, lookup_index); + env.insert(*ret, val.rep()); + } + HeadOp::Call { id, args, ret } => { + let argv: Vec = args + .iter() + .map(|s| resolve(s, env).map(Value::new)) + .collect::>()?; + let result = eg.eval_prim_internal(*id, &argv); + if let Some(v) = result { + env.insert(*ret, v.rep()); + } + // A `None` result (primitive failure) in an action is a no-op; + // real failures surface as panics through `PanicFunc`. + } + HeadOp::Union { .. } => { + // Term-encoding mode emits unions as `(set (@uf …))` writes, not + // trait `union` calls (mirrors the DuckDB/Feldera backends). A + // direct `union` therefore never reaches a tractable program. + } + HeadOp::Panic(msg) => { + return Err(anyhow!("{msg}")); + } + } + } + Ok(()) +} + +/// Build a full row from head-action slots under `env`. +fn build_row(slots: &[Slot], env: &Env) -> Result { + let vals: Vec = slots + .iter() + .map(|s| resolve(s, env)) + .collect::>()?; + Ok(vals.into_boxed_slice()) +} + +/// Resolve a slot to a concrete value, erroring if it is an unbound variable. +fn resolve(s: &Slot, env: &Env) -> Result { + slot_lookup(s, &|v| env.get(&v).copied()) + .ok_or_else(|| anyhow!("unbound variable {s:?} in rule head")) +} + +/// Look up the output of `func` for input `key`. If absent, create the row with +/// a fresh id (eq-sort constructor semantics — mirrors `add_term`). The created +/// row is written directly into the mirror so subsequent lookups in the same +/// iteration see it (hash-cons). +pub(crate) fn lookup_or_create( + eg: &mut EGraph, + func: FunctionId, + key: &[Value], + index: &mut HashMap, u32>>, +) -> Value { + let info = eg.info(func); + let inputs_len = info.arity.saturating_sub(1); + // Lazily build the key->output index for this function from the mirror so + // repeated lookups within one iteration are O(1) instead of O(state) scans. + let idx = index.entry(func).or_insert_with(|| { + let mut m: HashMap, u32> = HashMap::new(); + if let Some(set) = eg.mirror.get(&func) { + for row in set.iter() { + let k: Box<[u32]> = (0..inputs_len).map(|i| row_col(row, i)).collect(); + m.insert(k, row_col(row, inputs_len)); + } + } + m + }); + let k: Box<[u32]> = key.iter().map(|v| v.rep()).collect(); + if let Some(&out) = idx.get(&k) { + return Value::new(out); + } + let id = eg.fresh_id_internal(); + idx.insert(k, id); + let mut full: Vec = key.iter().map(|v| v.rep()).collect(); + full.push(id); + let row: Row = full.into_boxed_slice(); + std::rc::Rc::make_mut(eg.mirror.entry(func).or_default()).insert(row); + Value::new(id) +} diff --git a/egglog-experimental/flowlog/src/lib.rs b/egglog-experimental/flowlog/src/lib.rs new file mode 100644 index 0000000..2cb2c3c --- /dev/null +++ b/egglog-experimental/flowlog/src/lib.rs @@ -0,0 +1,1042 @@ +//! # egglog-bridge-flowlog +//! +//! A FlowLog (Differential-Dataflow)-backed executor for egglog's resolved IR, +//! behind the [`egglog_backend_trait::Backend`] interface. This is the FlowLog +//! analog of the Feldera/DBSP backend's Milestone 1. +//! +//! ## Milestone 1 — bounded per-iteration `(run N)` stepping on real flowlog-rs +//! +//! egglog's `(run N)` applies a ruleset **N times with bounded extension per +//! round** — a transitive-closure rule extends **N hops, NOT to full closure**. +//! This backend proves that bounded behavior runs *through the `Backend` trait* +//! on a **live, in-process flowlog-rs `DatalogIncrementalEngine`** (Differential +//! Dataflow), and matches the reference backend (`egglog_bridge::EGraph`) +//! round-for-round. +//! +//! The load-bearing mapping: **one `run_rules` call = one flowlog `commit()` = +//! one hop.** The bundled `transitive_step.dl` is **non-recursive** +//! (`hop(x,z) :- path(x,y), edge(y,z).`), so a single `commit()` performs +//! exactly one round of the join over the freshly-staged `path` delta. The host +//! folds each epoch's `hop` deltas into a Rust-side materialized mirror and +//! re-stages the new `path` rows for the next round (the PLAN §4 design-A +//! mirror + the proven Feldera host-feedback loop). N calls = N hops, bounded. +//! +//! ## The build-time-fixed `.dl` (and the FlowLog crux) +//! +//! flowlog compiles `.dl` -> Rust at BUILD time (`build.rs`), but egglog defines +//! rules at RUNTIME. For the M1 PROOF a build-time-fixed `.dl` is acceptable +//! (per the brief). `run_rules` therefore **recognizes** the canonical +//! transitive-closure-step rule shape the frontend builds and routes it to the +//! bundled engine. Runtime rule installation (the FlowLog crux — the analog of +//! Feldera's static-circuit-rebuild risk, but harder) is investigated in +//! ../MILESTONE1.md and deferred to M2. +//! +//! ## Read-back through the trait +//! +//! `for_each` / `for_each_while` / `lookup_id` / `table_size` read the +//! materialized mirror, refreshed from `commit()`'s per-epoch deltas. + +use std::any::Any; + +use anyhow::Result; +use egglog_backend_trait::{ + Backend, BaseValues, ColumnTy, ContainerValues, ExecutionState, ExternalFunction, + ExternalFunctionId, FunctionConfig, FunctionId, IterationReport, MergeFn, ReportLevel, + RuleBuilderOps, RuleId, ScanEntry, Value, +}; +use egglog_core_relations::Database; +use egglog_numeric_id::NumericId; +use hashbrown::{HashMap, HashSet}; + +pub mod codegen; +pub mod compile; +pub mod dd_native; +mod engine; +mod external_func; +pub mod interpret; +mod rule_builder; +pub mod subprocess; + +use compile::{row_col, unpack_row, BodyOp, HeadOp, MergeMode, Row, RuleIr, Slot}; +use external_func::ExternalFuncRegistry; + +// --------------------------------------------------------------------------- +// Relation metadata +// --------------------------------------------------------------------------- + +/// What we remember about each registered relation/function. +pub(crate) struct RelationInfo { + #[allow(dead_code)] + name: String, + /// Number of columns (including the output column for functions). + pub(crate) arity: usize, + /// True for functions/constructors that have an output column. + #[allow(dead_code)] + has_output: bool, + /// How functional-dependency conflicts are resolved. + pub(crate) merge: MergeMode, +} + +// --------------------------------------------------------------------------- +// EGraph +// --------------------------------------------------------------------------- + +/// How `run_rules` executes the recognized step. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExecMode { + /// M1: drive a build-time-fixed in-process flowlog engine (recognized + /// transitive-closure step only; used by `run_n_proof`). + InProcess, + /// M2: translate the recognized step to `.dl`, compile a driver crate + /// (cached by rule-set hash), and drive it as a subprocess over a pipe + /// (used by `run_n_shellout_proof`). + ShellOut, + /// M3 (the frontend default): the in-process iteration driver + /// ([`interpret::run_iteration`]) runs general multi-atom bodies + + /// primitives + head actions. Every body table-atom join runs on the + /// in-process Differential-Dataflow dataflow ([`dd_native`]); the primitive + /// tail + head actions are applied host-side. This is what real `.egg` files + /// run on. There is NO host nested-loop fallback — an unsupported rule shape + /// panics with a reason. + Interpret, +} + +/// The FlowLog-backed egraph. +pub struct EGraph { + relations: Vec, + /// Rule slots; `None` = freed. + pub(crate) rules: Vec>, + /// Rust-side materialized mirror: the accumulated contents of each + /// relation, kept in sync with the flowlog engine's per-epoch `commit()` + /// deltas. This is what `for_each` / `lookup_id` / `table_size` read. + /// + /// The set per function is shared by `Rc` so the per-iteration read + /// snapshot (see `interpret::run_iteration`) is O(#functions), not + /// O(state): mutations copy-on-write via `Rc::make_mut` only the functions + /// actually changed while a snapshot is alive (the Feldera flatness model). + pub(crate) mirror: HashMap>>, + /// The live, in-process flowlog-rs `DatalogIncrementalEngine`, wrapped so + /// the trait code stays free of the generated-symbol details. `None` until + /// the first transitive-closure-step `run_rules`, which is where we know + /// which `FunctionId`s play the `edge` / `path` / head roles. + flow: Option, + /// Execution mode: in-process flowlog engine (M1) or the M2 shell-out + /// driver subprocess (runtime rule installation via codegen + compile). + mode: ExecMode, + /// The M2 shell-out driver: a subprocess that embeds a flowlog engine + /// compiled at runtime from the rule IR. `None` until the first + /// `run_rules` under [`ExecMode::ShellOut`] (where the rule is known). + driver: Option, + /// Per-round feedback buffer for the shell-out path: the new `path` rows + /// derived last round, to re-stage as the next round's `insert path` delta + /// (the bounded host-feedback loop, mirrored across the pipe). + pending_path: Vec<(i32, i32)>, + /// A core-relations [`Database`] used purely as the base-value / primitive + /// engine, so `Value`s are bit-for-bit identical to the reference backend. + db: Database, + pub(crate) external_funcs: ExternalFuncRegistry, + /// Monotonic fresh-id counter for `fresh_id` / `add_term`. + pub(crate) next_id: u32, + report_level: ReportLevel, + /// Diagnostics: rule firings whose body join ran on the DD engine. + pub(crate) dd_rule_runs: u64, + /// Atom-less rules (`(rule () …)`) fire ONCE; an entry here marks a rule + /// index as already fired. The DD dataflow has no input relation to drive an + /// atom-less body, so this fired-marker is the one piece of seminaive + /// bookkeeping the DD path reuses (see `interpret::dd_native_bindings`). + /// `free_rule` removes the entry so a re-installed rule can fire again. + pub(crate) seen: HashMap, + /// Per-rule persistent DD join, built once (lazily) and stepped each + /// iteration with only the per-relation delta (the Feldera `persistent` analog). + pub(crate) dd_native: HashMap, + /// Per-rule, per-function last-fed row snapshot `Rc`, for computing the + /// signed delta vs what the persistent DD join was last fed (Feldera `fed`). + pub(crate) dd_native_fed: HashMap>>>, + /// Per-RULESET fused DD join (one shared timely worker + one dataflow for the + /// whole ruleset), keyed by the sorted live rule-index list — the FlowLog + /// analog of feldera's `FusedJoin`. This is the perf-critical join path; + /// `dd_native` (per-rule) is retained only for the bridge-level unit tests. + pub(crate) dd_fused: HashMap, dd_native::FusedDdJoin>, + /// Per-ruleset, per-function last-fed row snapshot `Rc`, for computing the + /// signed delta fed into the FUSED join's shared inputs (the fused `fed`). + pub(crate) dd_fused_fed: HashMap, HashMap>>>, + /// `--wcoj`: route the reverse-distributivity triangle rule through the + /// worst-case-optimal delta-query join (dogsdogsdogs prefix-extension + + /// AltNeu 3-stream decomposition in [`dd_native`]) instead of the left-deep + /// binary `.join` chain. Detected structurally per rule at fused-build time; + /// non-triangle rules are unaffected. Off by default — when off the FlowLog + /// join path is byte-identical to the pre-WCOJ behavior. + pub(crate) wcoj_enabled: bool, +} + +impl Default for EGraph { + fn default() -> Self { + Self::new() + } +} + +impl Drop for EGraph { + fn drop(&mut self) { + // Step-0 profile dump (gated FLOWLOG_DD_PROF): #workers, #InputSessions, + // and the worker.step vs host-side prim/delta split. No-op otherwise. + dd_native::dd_prof_dump(); + // Per-ruleset profile dump (gated FLOWLOG_DD_RULESET_PROF): attribute + // DD wall time to the ruleset NAME, sorted descending. No-op otherwise. + dd_native::dd_ruleset_prof_dump(); + } +} + +impl EGraph { + /// Construct a fresh FlowLog-backed egraph (M1 in-process mode). + pub fn new() -> Self { + Self::with_mode(ExecMode::InProcess) + } + + /// Construct a fresh FlowLog-backed egraph driven by the M2 shell-out + /// runtime-codegen path (compile a driver subprocess from the rule IR). + pub fn new_shellout() -> Self { + Self::with_mode(ExecMode::ShellOut) + } + + /// Construct a fresh FlowLog-backed egraph driven by the M3 iteration driver + /// (general bodies + primitives + head actions; every body table-atom join + /// runs on the in-process Differential-Dataflow dataflow). This is the + /// constructor the egglog frontend uses (`EGraph::with_flowlog_backend`). + pub fn new_interpret() -> Self { + Self::with_mode(ExecMode::Interpret) + } + + /// Construct with an explicit execution mode. + pub fn with_mode(mode: ExecMode) -> Self { + let mut db = Database::new(); + // Pre-register the `()` (Unit) base type so `add_table`'s relation-vs- + // function detection can always resolve the Unit `BaseValueId`. + // `register_type` is idempotent, so a later frontend registration is a + // no-op that returns the same id. + db.base_values_mut().register_type::<()>(); + EGraph { + relations: Vec::new(), + rules: Vec::new(), + mirror: HashMap::new(), + flow: None, + mode, + driver: None, + pending_path: Vec::new(), + db, + external_funcs: ExternalFuncRegistry::default(), + // Start at 1 so id 0 stays a "null"/padding sentinel. + next_id: 1, + report_level: ReportLevel::default(), + dd_rule_runs: 0, + seen: HashMap::new(), + dd_native: HashMap::new(), + dd_native_fed: HashMap::new(), + dd_fused: HashMap::new(), + dd_fused_fed: HashMap::new(), + wcoj_enabled: false, + } + } + + /// Turn on `--wcoj`: route the reverse-distributivity triangle rule through + /// the worst-case-optimal delta-query join (see [`dd_native`]). When off, + /// every rule lowers to the left-deep binary `.join` chain exactly as + /// before. Detected structurally at fused-build time; non-triangle rules are + /// unaffected. Off by default. + pub fn enable_wcoj(&mut self) { + self.wcoj_enabled = true; + } + + pub(crate) fn info(&self, f: FunctionId) -> &RelationInfo { + self.relations + .get(f.rep() as usize) + .unwrap_or_else(|| panic!("FunctionId({}) not registered", f.rep())) + } + + /// Relation name for `f` (used by the per-ruleset profiler to detect `@uf` + /// body atoms — the union-find tables are named `@UF_` / `@UF_f`). + pub(crate) fn relation_name(&self, f: FunctionId) -> &str { + &self.info(f).name + } + + /// Inherent accessor for the embedded [`BaseValues`] registry (the frontend + /// extraction path threads `&BaseValues` through `reconstruct_termdag_base`). + pub fn base_values_inner(&self) -> &egglog_core_relations::BaseValues { + self.db.base_values() + } + + /// Inherent accessor for the embedded [`Database`]'s container registry, so + /// the frontend extraction path can read interned container values. + pub fn container_values_inner(&self) -> &egglog_core_relations::ContainerValues { + self.db.container_values() + } + + /// Diagnostics: the number of rule firings whose body table-atom join ran on + /// the in-process Differential-Dataflow dataflow. Every atom-bearing rule + /// runs there (no host fallback); lets a test assert the join genuinely ran + /// on DD. + pub fn flowlog_dd_rule_runs(&self) -> u64 { + self.dd_rule_runs + } + + /// The functional-dependency merge mode of a function (from `add_table`). + pub(crate) fn merge_mode(&self, f: FunctionId) -> MergeMode { + self.info(f).merge + } + + /// Evaluate a primitive through the embedded `Database` (the base-value / + /// primitive engine). Both the host interpreter and the DD-join path's + /// host-side primitive tail call this, so `Value`s are bit-for-bit + /// identical to the reference backend. This is the inherent counterpart of + /// Feldera M4's `eval_prim` trait method — flowlog keeps it inherent to + /// preserve its zero-trait-change posture (no `egglog-backend-trait` edit). + pub(crate) fn eval_prim_internal( + &self, + id: ExternalFunctionId, + args: &[Value], + ) -> Option { + self.db + .with_execution_state(|st| st.call_external_func(id, args)) + } + + /// Allocate a fresh id (the interpreter's eq-sort constructor hash-cons uses + /// the same counter the trait's `fresh_id` advances). + pub(crate) fn fresh_id_internal(&mut self) -> u32 { + let id = self.next_id; + self.next_id += 1; + id + } + + /// Resolve functional-dependency conflicts in a function's mirror set: for + /// each key (input columns) keep a single output column chosen by the merge + /// mode. Relations (whole-row key) are left untouched. Returns whether any + /// row was actually changed/collapsed. + /// + /// INCREMENTAL (ported from Feldera's flatness work): the pre-call mirror is + /// already FD-resolved (this runs every `run_rules` call), so only the keys + /// whose rows were `set` this call (in `keys`) can newly conflict. We + /// re-resolve just those keys instead of rebuilding the whole function — + /// O(touched-keys), not O(state). The deterministic fold order (sort) is + /// preserved PER KEY, which gives the same chosen output as the old + /// whole-set sort+fold (the fold is independent across distinct keys), so + /// `Old`/`New` conflicts arriving in the same iteration resolve stably. + pub(crate) fn resolve_merge(&mut self, f: FunctionId, keys: &HashSet>) -> bool { + let arity = self.info(f).arity; + let merge = self.merge_mode(f); + if !matches!(merge, MergeMode::Old | MergeMode::New | MergeMode::Min) + || arity == 0 + || keys.is_empty() + { + return false; + } + let Some(set) = self.mirror.get(&f) else { + return false; + }; + let inputs_len = arity - 1; + // Gather the candidate rows for the touched keys only. + let mut by_key: HashMap<&[u32], Vec<&Row>> = HashMap::new(); + for row in set.iter() { + let key: Vec = (0..inputs_len).map(|i| row_col(row, i)).collect(); + if keys.contains(&key) { + by_key.entry(&row[..inputs_len]).or_default().push(row); + } + } + // FlowLog runs the fast RELATIONAL term-encoding: congruence is RULE-ENCODED + // (`@congruence_rule*`) and canonicalized by the `@rebuild_rule*` rules on the + // DD engine. So an ordinary function's FD conflict resolves by the plain + // merge mode (Min/Old/New) — there is no in-core-UF congruence injection + // (non-proof `--native-merge --flowlog` is rejected; proof-mode native-merge + // routes through `native_merge_proof` above). Resolve each touched key; + // collect the rows to remove and the winner to insert. Only keys with >1 + // candidate row can change. + let mut new_rows: Vec = Vec::new(); + let mut drop_rows: HashSet = HashSet::new(); + for (_key, mut cands) in by_key { + if cands.len() < 2 { + continue; + } + // Deterministic fold order (mirror is a HashSet — arbitrary order). + cands.sort(); + let mut chosen = row_col(cands[0], inputs_len); + for row in &cands[1..] { + let out = row_col(row, inputs_len); + chosen = match merge { + MergeMode::Old => chosen, + MergeMode::New => out, + MergeMode::Min => chosen.min(out), + MergeMode::Relation => unreachable!(), + }; + } + // The winning row. + let mut winner: Vec = cands[0][..inputs_len].to_vec(); + winner.push(chosen); + let winner: Row = winner.into_boxed_slice(); + for row in &cands { + if ***row != *winner { + drop_rows.insert((**row).clone()); + } + } + new_rows.push(winner); + } + if drop_rows.is_empty() { + return false; + } + let set = std::rc::Rc::make_mut(self.mirror.get_mut(&f).unwrap()); + for r in &drop_rows { + set.remove(r); + } + for r in &new_rows { + set.insert(r.clone()); + } + true + } +} + +// --------------------------------------------------------------------------- +// Send + Sync (single-threaded use; same posture as DuckDB/Feldera) +// --------------------------------------------------------------------------- +// +// The flowlog engine owns a timely worker `JoinHandle` and channel endpoints, +// which are not all auto-`Sync`. As with the sibling backends, the egraph is +// only ever driven from a single thread, so we assert the bounds the trait +// requires. Concurrent multi-thread use is unsupported. +unsafe impl Send for EGraph {} +unsafe impl Sync for EGraph {} + +// --------------------------------------------------------------------------- +// Merge-mode recognition (shared by `add_table`) +// --------------------------------------------------------------------------- + +/// Map a single-output-column [`MergeFn`] to the FlowLog [`MergeMode`] that +/// resolves that column's FD conflict: +/// - `AssertEq` / `Old` => keep the old value +/// - `New` => keep the new value +/// - `UnionId` => lattice-min (the union-find leader) +/// - `Primitive(_)` => lattice-min (the term encoder's `:merge ordering-min`) +/// - `Function` / `Const` => fall back to `Old` +fn merge_mode_for_scalar(merge: &MergeFn) -> MergeMode { + match merge { + MergeFn::AssertEq | MergeFn::Old => MergeMode::Old, + MergeFn::New => MergeMode::New, + MergeFn::UnionId | MergeFn::Primitive(_, _) => MergeMode::Min, + MergeFn::Function(_, _) | MergeFn::Const(_) => MergeMode::Old, + } +} + +// --------------------------------------------------------------------------- +// impl Backend +// --------------------------------------------------------------------------- + +impl Backend for EGraph { + // -- table lifecycle ---------------------------------------------------- + + fn add_table(&mut self, config: FunctionConfig) -> FunctionId { + let id = FunctionId::new(self.relations.len() as u32); + let arity = config.schema.len(); + assert!( + arity <= compile::MAX_ARITY, + "FlowLog backend supports relations of arity <= {} (got {} for `{}`)", + compile::MAX_ARITY, + arity, + config.name + ); + // Relation vs function: a table is a **relation** (whole row is the key, + // no output column to merge) iff it is nullary OR its last column is + // `Unit` — the term encoder's view-table pattern + // `(function @XView (...) Unit :merge old)` AND ordinary relations. + // Otherwise the last column is a function OUTPUT, resolved by the merge + // mode. This mirrors the DuckDB/Feldera backends' Unit-detection — NOT + // `DefaultVal`, which is `Fail` for every custom function regardless of + // whether it has a real output column. (This is the fix that took the + // Feldera survey from 38 to 59 passing files.) + let output_is_unit = config.schema.last().is_some_and(|t| match t { + ColumnTy::Base(bv) => { + // `()` is pre-registered in `with_mode`, so this never panics. + *bv == self + .db + .base_values() + .get_ty_by_id(std::any::TypeId::of::<()>()) + } + _ => false, + }); + let has_output = arity > 0 && !output_is_unit; + let merge = if !has_output { + MergeMode::Relation + } else { + merge_mode_for_scalar(&config.merge) + }; + self.relations.push(RelationInfo { + name: config.name, + arity, + has_output, + merge, + }); + self.mirror.insert(id, std::rc::Rc::new(HashSet::new())); + id + } + + fn table_size(&self, table: FunctionId) -> usize { + self.mirror.get(&table).map(|s| s.len()).unwrap_or(0) + } + + // -- iteration ---------------------------------------------------------- + + fn for_each_dyn(&self, table: FunctionId, f: &mut dyn for<'r> FnMut(ScanEntry<'r>)) { + self.for_each_while_dyn(table, &mut |row| { + f(row); + true + }); + } + + fn for_each_while_dyn( + &self, + table: FunctionId, + f: &mut dyn for<'r> FnMut(ScanEntry<'r>) -> bool, + ) { + let arity = self.info(table).arity; + let Some(set) = self.mirror.get(&table) else { + return; + }; + for row in set.iter() { + let vals = unpack_row(row, arity); + let entry = ScanEntry { + vals: &vals, + subsumed: false, + }; + if !f(entry) { + break; + } + } + } + + // -- direct access ------------------------------------------------------ + + fn get_canon_repr(&self, val: Value, _ty: ColumnTy) -> Value { + // No union-find; canonicalization is the identity. + val + } + + fn clear_table(&mut self, func: FunctionId) { + if let Some(set) = self.mirror.get_mut(&func) { + std::rc::Rc::make_mut(set).clear(); + } + // The DD `fed` snapshots (`dd_native_fed` / `dd_fused_fed`) are diffed + // against the live mirror each iteration, so clearing the mirror is + // picked up as a retraction delta automatically — no extra bookkeeping + // needed here. + } + + fn base_values(&self) -> &BaseValues { + self.db.base_values() + } + + fn base_values_mut(&mut self) -> &mut BaseValues { + self.db.base_values_mut() + } + + fn container_values(&self) -> &ContainerValues { + self.db.container_values() + } + + fn with_execution_state_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)) { + self.db.with_execution_state(|st| f(st)); + } + + fn with_execution_state_tracked_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)) -> bool { + self.db.with_execution_state_tracked(|st| f(st)).1 + } + + // -- rule management ---------------------------------------------------- + + fn new_rule<'a>(&'a mut self, desc: &str, _seminaive: bool) -> Box { + // Seminaive is native to differential dataflow; the flag is accepted + // for parity and ignored. + Box::new(rule_builder::FlowlogRuleBuilder::new(self, desc)) + } + + fn free_rule(&mut self, id: RuleId) { + if let Some(slot) = self.rules.get_mut(id.rep() as usize) { + *slot = None; + let i = id.rep() as usize; + self.seen.remove(&i); + self.dd_native.remove(&i); + self.dd_native_fed.remove(&i); + // Any fused ruleset that included this rule is now stale: drop it so + // it is rebuilt (without the freed rule) on the next `run_rules`. + self.dd_fused.retain(|key, _| !key.contains(&i)); + self.dd_fused_fed.retain(|key, _| !key.contains(&i)); + } + } + + fn run_rules(&mut self, rules: &[RuleId]) -> Result { + // ONE egglog iteration = ONE flowlog `commit()` = ONE transitive- + // closure hop. The frontend calls this N times for `(run N)`. + if rules.is_empty() { + return Ok(IterationReport::default()); + } + let live: Vec = rules + .iter() + .map(|r| r.rep() as usize) + .filter(|&i| self.rules.get(i).map(|s| s.is_some()).unwrap_or(false)) + .collect(); + if live.is_empty() { + return Ok(IterationReport::default()); + } + + let changed = match self.mode { + ExecMode::Interpret => interpret::run_iteration(self, &live)?, + ExecMode::InProcess | ExecMode::ShellOut => self.run_one_hop(&live)?, + }; + + let mut report = IterationReport::default(); + report.rule_set_report.changed = changed; + Ok(report) + } + + fn flush_updates(&mut self) -> bool { + // Seed inserts land in the mirror immediately; flowlog staging happens + // inside `run_rules` (one commit per call). No separate flush. + false + } + + // -- primitives --------------------------------------------------------- + + fn register_external_func( + &mut self, + func: Box, + ) -> ExternalFunctionId { + let func2 = dyn_clone::clone_box(&*func); + let id = self.db.add_external_function(func); + self.external_funcs.add_func_at(id, func2); + id + } + + fn free_external_func(&mut self, func: ExternalFunctionId) { + self.db.free_external_function(func); + self.external_funcs.free(func); + } + + fn new_panic(&mut self, message: String) -> ExternalFunctionId { + let panic_fn = external_func::PanicFunc::new(message.clone()); + let id = self.db.add_external_function(Box::new(panic_fn)); + self.external_funcs.add_panic_at(id, message); + id + } + + // -- capability flags --------------------------------------------------- + + fn supports_subsumption(&self) -> bool { + false + } + + fn supports_containers(&self) -> bool { + false + } + + fn supports_action_registry(&self) -> bool { + // No egglog `ActionRegistry`. Registry-backed primitives are registered + // via the frontend's registry-free placeholder/snapshot path and + // dispatched by the FlowLog interpreter's external-func mechanism. + false + } + + // -- diagnostics -------------------------------------------------------- + + fn set_report_level(&mut self, level: ReportLevel) { + self.report_level = level; + } + + fn dump_debug_info(&self) { + for (i, info) in self.relations.iter().enumerate() { + let f = FunctionId::new(i as u32); + let n = self.table_size(f); + if n == 0 { + continue; + } + log::info!("== FlowLog relation `{}` ({} rows) ==", info.name, n); + } + } + + // -- cloning ------------------------------------------------------------ + + fn clone_boxed(&self) -> Box { + // Push/pop snapshot support is a later milestone: a running flowlog + // dataflow can't be cloned, but the *state* (mirror + rule IR + + // relation metadata) can be replayed into a fresh engine. Not needed + // for milestone 1. + unimplemented!( + "FlowLog backend clone_boxed (push/pop) is deferred (snapshot-and-replay \ + into a fresh engine)" + ) + } + + // -- bridge-only escape hatch ------------------------------------------ + + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } +} + +// --------------------------------------------------------------------------- +// The flowlog-driven per-iteration hop +// --------------------------------------------------------------------------- + +/// The roles played by the three `FunctionId`s in a transitive-closure-step +/// rule `head(x, z) :- path(x, y), edge(y, z)`, recognized from the rule IR. +struct StepShape { + /// The body atom whose middle (join) column also appears in the *other* + /// body atom's first column — i.e. the "left/path" relation contributing + /// the head's first column. + path: FunctionId, + /// The "right/edge" relation contributing the head's last column. + edge: FunctionId, + /// The head relation (where new rows are derived). In egglog's + /// transitive-closure step this equals `path`. + head: FunctionId, + /// Column index in `path` rows of the head's first variable (`x`). + path_x: usize, + /// Column index in `path` rows of the shared join variable (`y`). + path_y: usize, + /// Column index in `edge` rows of the shared join variable (`y`). + edge_y: usize, + /// Column index in `edge` rows of the head's last variable (`z`). + edge_z: usize, + /// The (constant) value to write in any non-(x,z) head columns, by column + /// index — e.g. the fixed FD value of a `(x,y) -> value` table. + head_consts: Vec<(usize, u32)>, + /// Which head column carries `x` and which carries `z`. + head_x: usize, + head_z: usize, + /// Head arity. + head_arity: usize, +} + +impl EGraph { + /// Run exactly one transitive-closure hop through the bundled flowlog + /// incremental engine: stage this round's new `path` rows, `commit()` once, + /// fold the resulting `hop` deltas into the mirror. Returns whether the + /// mirror changed. + fn run_one_hop(&mut self, live: &[usize]) -> Result { + match self.mode { + ExecMode::InProcess => self.run_one_hop_inprocess(live), + ExecMode::ShellOut => self.run_one_hop_shellout(live), + ExecMode::Interpret => { + unreachable!("Interpret mode routes through interpret::run_iteration") + } + } + } + + /// M1 in-process path: drive the build-time-fixed flowlog engine. + fn run_one_hop_inprocess(&mut self, live: &[usize]) -> Result { + // Exactly one rule per `run_rules`: the transitive-closure step. (The + // frontend's `(run N)` loop calls this with the single user rule N + // times.) + let shape = self.recognize_step(live)?; + + // Lazily build the engine and do the initial seeding hop. On the first + // call we feed ALL current `edge` and `path` rows; the non-recursive + // join's first commit yields the 1-hop extension. On subsequent calls + // we feed only the previous round's NEW path rows so each commit is + // exactly one further hop (bounded extension, not saturation). + if self.flow.is_none() { + self.flow = Some(engine::FlowEngine::new()); + // Seed all edges (they never change across rounds in this proof). + let edge_rows = self.engine_rows(shape.edge, shape.edge_y, shape.edge_z); + // Seed all current path rows as the first delta. + let path_rows = self.engine_rows(shape.path, shape.path_x, shape.path_y); + let new_hops = { + let flow = self.flow.as_mut().unwrap(); + flow.insert_edge(&edge_rows); + flow.insert_path(&path_rows); + flow.commit_hop() + }; + return Ok(self.fold_hops(&shape, &new_hops)); + } + + // Subsequent rounds: re-stage the rows derived LAST round as the new + // `path` delta (tracked in the engine wrapper), commit one hop, fold. + let to_feed = self.flow.as_mut().unwrap().take_pending_path(); + if to_feed.is_empty() { + // Nothing new to extend with: no further hop is possible. + return Ok(false); + } + let new_hops = { + let flow = self.flow.as_mut().unwrap(); + flow.insert_path(&to_feed); + flow.commit_hop() + }; + Ok(self.fold_hops(&shape, &new_hops)) + } + + /// M2 shell-out path: translate the runtime rule to `.dl`, compile (or + /// reuse a cached) driver subprocess, and drive ONE bounded hop over the + /// pipe. Same host-feedback loop as the in-process path, but the engine + /// lives in a subprocess compiled from the rule defined at runtime. + fn run_one_hop_shellout(&mut self, live: &[usize]) -> Result { + let shape = self.recognize_step(live)?; + + // First call: emit the `.dl` from the runtime rule, build/cache + spawn + // the driver, then stage all current `edge` + `path` rows and commit + // the first (1-hop) epoch. + if self.driver.is_none() { + let dl = codegen::emit_dl(); + let mut handle = subprocess::DriverHandle::build_or_cached(&dl)?; + handle.spawn()?; + self.driver = Some(handle); + + let edge_rows = self.engine_rows(shape.edge, shape.edge_y, shape.edge_z); + let path_rows = self.engine_rows(shape.path, shape.path_x, shape.path_y); + let new_hops = { + let drv = self.driver.as_mut().unwrap(); + for (a, b) in &edge_rows { + drv.insert(codegen::REL_EDGE, *a, *b)?; + } + for (a, b) in &path_rows { + drv.insert(codegen::REL_PATH, *a, *b)?; + } + drv.commit()? + }; + return Ok(self.fold_hops_shellout(&shape, &new_hops)); + } + + // Subsequent rounds: re-stage last round's NEW path rows as the next + // `insert path` delta, commit one further hop, fold. + let to_feed = std::mem::take(&mut self.pending_path); + if to_feed.is_empty() { + return Ok(false); + } + let new_hops = { + let drv = self.driver.as_mut().unwrap(); + for (a, b) in &to_feed { + drv.insert(codegen::REL_PATH, *a, *b)?; + } + drv.commit()? + }; + Ok(self.fold_hops_shellout(&shape, &new_hops)) + } + + /// Fold the shell-out driver's `hop` deltas into the mirror and stage the + /// new `path` rows for next round. Mirrors `fold_hops` but writes the + /// feedback buffer on the egraph (the subprocess is stateless across + /// rounds w.r.t. host feedback). + fn fold_hops_shellout(&mut self, shape: &StepShape, hops: &[(i32, i32, i32)]) -> bool { + let mut changed = false; + let mut new_path_feed: Vec<(i32, i32)> = Vec::new(); + for &(x, z, diff) in hops { + if diff <= 0 { + continue; + } + let mut full = vec![0u32; shape.head_arity]; + for &(ci, cv) in &shape.head_consts { + full[ci] = cv; + } + full[shape.head_x] = x as u32; + full[shape.head_z] = z as u32; + let row: Row = full.into_boxed_slice(); + let set = std::rc::Rc::make_mut(self.mirror.entry(shape.head).or_default()); + if set.insert(row) { + changed = true; + new_path_feed.push((x, z)); + } + } + self.pending_path = new_path_feed; + changed + } + + /// Collect the `(a, b)` projection of relation `f`'s mirror rows at the + /// given two column indices, as engine tuples. + fn engine_rows(&self, f: FunctionId, ca: usize, cb: usize) -> Vec<(i32, i32)> { + let mut out = Vec::new(); + if let Some(set) = self.mirror.get(&f) { + for row in set.iter() { + out.push((row_col(row, ca) as i32, row_col(row, cb) as i32)); + } + } + out + } + + /// Fold this epoch's `hop` deltas `(x, z)` into the head-relation mirror. + /// Records the genuinely-new head rows so the *next* round can feed them + /// back to the engine as the next `path` delta. Returns whether the mirror + /// gained any rows. + fn fold_hops(&mut self, shape: &StepShape, hops: &[(i32, i32, i32)]) -> bool { + let mut changed = false; + let mut new_path_feed: Vec<(i32, i32)> = Vec::new(); + for &(x, z, diff) in hops { + if diff <= 0 { + // M1 is monotone (no retraction); ignore non-positive diffs. + continue; + } + // Build the full head row: x and z in their columns, constants + // elsewhere. + let mut full = vec![0u32; shape.head_arity]; + for &(ci, cv) in &shape.head_consts { + full[ci] = cv; + } + full[shape.head_x] = x as u32; + full[shape.head_z] = z as u32; + let row: Row = full.into_boxed_slice(); + let set = std::rc::Rc::make_mut(self.mirror.entry(shape.head).or_default()); + if set.insert(row) { + changed = true; + // The head IS the path relation in a transitive-closure step, + // so a new head row is a new path row to extend with next round. + // Feed its (x, y=z) projection — for path the join column is + // path_y, and the new row's path_y position holds `z`. + new_path_feed.push((x, z)); + } + } + if let Some(flow) = self.flow.as_mut() { + flow.set_pending_path(new_path_feed); + } + changed + } + + /// Recognize the transitive-closure-step shape from the single live rule. + fn recognize_step(&self, live: &[usize]) -> Result { + if live.len() != 1 { + return Err(anyhow::anyhow!( + "FlowLog backend (M1) runs exactly one rule per `run_rules` \ + (got {}); multi-rule rulesets are an M2 feature.", + live.len() + )); + } + let ir = self.rules[live[0]] + .as_ref() + .ok_or_else(|| anyhow::anyhow!("FlowLog backend: rule slot freed"))?; + + // Body: exactly two table atoms. + let mut atoms = Vec::new(); + for op in &ir.body { + match op { + BodyOp::Atom(a) => atoms.push(a), + BodyOp::Prim { .. } => { + return Err(anyhow::anyhow!( + "FlowLog backend (M1): primitive body atoms are not \ + supported (rule `{}`)", + ir.name + )) + } + } + } + if atoms.len() != 2 { + return Err(anyhow::anyhow!( + "FlowLog backend (M1) supports a two-atom join body \ + (transitive-closure step); rule `{}` has {} body atoms.", + ir.name, + atoms.len() + )); + } + // Head: exactly one `set`. + let set_head = ir.head.iter().find_map(|h| match h { + HeadOp::Set { func, slots } => Some((*func, slots)), + _ => None, + }); + let (head_func, head_slots) = set_head.ok_or_else(|| { + anyhow::anyhow!("FlowLog backend (M1): rule `{}` has no `set` head", ir.name) + })?; + + // Identify head variables (x = first, z = last) and constant columns. + let head_vars: Vec> = head_slots + .iter() + .map(|s| match s { + Slot::Var(v) => Some(*v), + Slot::Const(_) => None, + }) + .collect(); + let head_var_positions: Vec = head_vars + .iter() + .enumerate() + .filter_map(|(i, v)| v.map(|_| i)) + .collect(); + if head_var_positions.len() != 2 { + return Err(anyhow::anyhow!( + "FlowLog backend (M1): head must bind exactly two variables \ + (x, z); rule `{}`", + ir.name + )); + } + let head_x = head_var_positions[0]; + let head_z = head_var_positions[1]; + let var_x = head_vars[head_x].unwrap(); + let var_z = head_vars[head_z].unwrap(); + let head_consts: Vec<(usize, u32)> = head_slots + .iter() + .enumerate() + .filter_map(|(i, s)| match s { + Slot::Const(c) => Some((i, *c)), + Slot::Var(_) => None, + }) + .collect(); + let head_arity = head_slots.len(); + + // Find which atom carries `x` (-> path) and which carries `z` (-> edge). + let atom_has = |a: &compile::BodyAtom, var: u32| -> Option { + a.slots + .iter() + .position(|s| matches!(s, Slot::Var(v) if *v == var)) + }; + // path atom contains x; edge atom contains z. + let (path_atom, edge_atom) = + if atom_has(atoms[0], var_x).is_some() && atom_has(atoms[1], var_z).is_some() { + (atoms[0], atoms[1]) + } else if atom_has(atoms[1], var_x).is_some() && atom_has(atoms[0], var_z).is_some() { + (atoms[1], atoms[0]) + } else { + return Err(anyhow::anyhow!( + "FlowLog backend (M1): could not match head vars to body atoms \ + in rule `{}` (expected transitive-closure-step shape)", + ir.name + )); + }; + + let path_x = atom_has(path_atom, var_x).unwrap(); + let edge_z = atom_has(edge_atom, var_z).unwrap(); + + // The shared join variable `y` is the var that appears in BOTH atoms + // and is neither x nor z. + let path_vars: HashSet = path_atom + .slots + .iter() + .filter_map(|s| match s { + Slot::Var(v) => Some(*v), + _ => None, + }) + .collect(); + let mut join_var = None; + for s in &edge_atom.slots { + if let Slot::Var(v) = s { + if *v != var_x && *v != var_z && path_vars.contains(v) { + join_var = Some(*v); + break; + } + } + } + let join_var = join_var.ok_or_else(|| { + anyhow::anyhow!( + "FlowLog backend (M1): no shared join variable between body \ + atoms in rule `{}`", + ir.name + ) + })?; + let path_y = atom_has(path_atom, join_var).unwrap(); + let edge_y = atom_has(edge_atom, join_var).unwrap(); + + Ok(StepShape { + path: path_atom.func, + edge: edge_atom.func, + head: head_func, + path_x, + path_y, + edge_y, + edge_z, + head_consts, + head_x, + head_z, + head_arity, + }) + } +} diff --git a/egglog-experimental/flowlog/src/rule_builder.rs b/egglog-experimental/flowlog/src/rule_builder.rs new file mode 100644 index 0000000..5d57aed --- /dev/null +++ b/egglog-experimental/flowlog/src/rule_builder.rs @@ -0,0 +1,188 @@ +//! `impl RuleBuilderOps` for the FlowLog backend. +//! +//! Like the Feldera/DuckDB backends' rule builders, this is an **accumulator**: +//! each `RuleBuilderOps` call appends to an in-progress [`RuleIr`] (defined in +//! `compile.rs`) in emission order; `build()` registers it on the egraph. +//! +//! M1 records table body atoms and `set`/`insert` heads (the transitive-closure +//! step shape). The remaining ops (`query_prim`, `lookup`, `union`, `remove`, +//! `subsume`, `panic`, `call_external_func`) are accumulated into the IR for +//! later milestones but are not exercised by the M1 flowlog-driven path. + +use anyhow::Result; +use egglog_backend_trait::{ + Backend, BaseValues, ColumnTy, ExternalFunctionId, FunctionId, PanicMsg, QueryEntry, + RuleBuilderOps, RuleId, Variable, +}; +use egglog_numeric_id::NumericId; + +use crate::compile::{BodyAtom, BodyOp, HeadOp, RuleIr, Slot}; +use crate::EGraph; + +/// Accumulates a rule's body ops and head ops, then registers them. +pub struct FlowlogRuleBuilder<'a> { + egraph: &'a mut EGraph, + ir: RuleIr, + /// Fresh variable counter, seeded above any caller-provided variable id. + next_var: u32, + /// First error hit during accumulation; surfaced at `build()`. + deferred_err: Option, +} + +impl<'a> FlowlogRuleBuilder<'a> { + pub fn new(egraph: &'a mut EGraph, desc: &str) -> Self { + FlowlogRuleBuilder { + egraph, + ir: RuleIr { + name: desc.to_string(), + body: Vec::new(), + head: Vec::new(), + }, + next_var: 1 << 24, // keep builder-synthesized vars away from caller ids + deferred_err: None, + } + } + + fn fresh_var(&mut self, name: Option<&str>) -> QueryEntry { + let rep = self.next_var; + self.next_var += 1; + QueryEntry::Var(Variable { + // `VariableId` is not re-exported by the backend trait; the target + // type is inferred from the `id` field so `NumericId::new` mints it + // without naming the type. + id: NumericId::new(rep), + name: name.map(|s| s.to_string().into_boxed_str()), + }) + } +} + +impl<'a> RuleBuilderOps for FlowlogRuleBuilder<'a> { + fn new_var(&mut self, _ty: ColumnTy) -> QueryEntry { + self.fresh_var(None) + } + + fn new_var_named(&mut self, _ty: ColumnTy, name: &str) -> QueryEntry { + self.fresh_var(Some(name)) + } + + fn base_values(&self) -> &BaseValues { + self.egraph.base_values_inner() + } + + fn query_table( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + _is_subsumed: Option, + ) -> Result<()> { + let atom = BodyAtom::from_entries(func, entries); + self.ir.body.push(BodyOp::Atom(atom)); + Ok(()) + } + + fn query_prim( + &mut self, + func: ExternalFunctionId, + entries: &[QueryEntry], + _ret_ty: ColumnTy, + ) -> Result<()> { + if entries.is_empty() { + return Err(anyhow::anyhow!( + "FlowLog backend: query_prim with no entries" + )); + } + let args: Vec = entries[..entries.len() - 1] + .iter() + .map(Slot::from_entry) + .collect(); + let ret = Slot::from_entry(entries.last().unwrap()); + self.ir.body.push(BodyOp::Prim { + id: func, + args, + ret, + }); + Ok(()) + } + + fn call_external_func( + &mut self, + func: ExternalFunctionId, + args: &[QueryEntry], + _ret_ty: ColumnTy, + _panic_msg: PanicMsg, + ) -> QueryEntry { + let ret = self.fresh_var(None); + let QueryEntry::Var(Variable { id, .. }) = &ret else { + unreachable!() + }; + let rid = id.rep(); + let slots: Vec = args.iter().map(Slot::from_entry).collect(); + self.ir.head.push(HeadOp::Call { + id: func, + args: slots, + ret: rid, + }); + ret + } + + fn lookup( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + _panic_msg: PanicMsg, + ) -> QueryEntry { + let ret = self.fresh_var(None); + let QueryEntry::Var(Variable { id, .. }) = &ret else { + unreachable!() + }; + let rid = id.rep(); + let args: Vec = entries.iter().map(Slot::from_entry).collect(); + self.ir.head.push(HeadOp::Lookup { + func, + args, + ret: rid, + }); + ret + } + + fn subsume(&mut self, func: FunctionId, entries: &[QueryEntry]) -> Result<()> { + let slots: Vec = entries.iter().map(Slot::from_entry).collect(); + self.ir.head.push(HeadOp::Subsume { func, slots }); + Ok(()) + } + + fn set(&mut self, func: FunctionId, entries: &[QueryEntry]) { + let slots: Vec = entries.iter().map(Slot::from_entry).collect(); + self.ir.head.push(HeadOp::Set { func, slots }); + } + + fn remove(&mut self, func: FunctionId, entries: &[QueryEntry]) { + let slots: Vec = entries.iter().map(Slot::from_entry).collect(); + self.ir.head.push(HeadOp::Remove { func, slots }); + } + + fn union(&mut self, l: QueryEntry, r: QueryEntry) { + self.ir.head.push(HeadOp::Union { + l: Slot::from_entry(&l), + r: Slot::from_entry(&r), + }); + } + + fn panic(&mut self, message: String) { + self.ir.head.push(HeadOp::Panic(message)); + } + + fn new_panic(&mut self, message: String) -> ExternalFunctionId { + Backend::new_panic(self.egraph, message) + } + + fn build(self: Box) -> Result { + let this = *self; + if let Some(e) = this.deferred_err { + return Err(e); + } + let id = RuleId::new(this.egraph.rules.len() as u32); + this.egraph.rules.push(Some(this.ir)); + Ok(id) + } +} diff --git a/egglog-experimental/flowlog/src/subprocess.rs b/egglog-experimental/flowlog/src/subprocess.rs new file mode 100644 index 0000000..7792819 --- /dev/null +++ b/egglog-experimental/flowlog/src/subprocess.rs @@ -0,0 +1,385 @@ +//! The M2 shell-out mechanism: compile a runtime-emitted driver crate, cache +//! the built binary by rule-set hash, spawn it once, and drive its flowlog +//! incremental engine over a line-based stdin/stdout pipe. +//! +//! ## Lifecycle +//! +//! 1. `DriverHandle::build_or_cached(dl)` — given the runtime `.dl` text: hash +//! it (FNV-1a) to a stable cache key; if `$cache//driver` already +//! exists, reuse it (instant); else materialize a temp crate (`Cargo.toml` + +//! `build.rs` + `src/main.rs` + `program.dl`) under `$cache//` and +//! shell out to `cargo build --release` (cold builds compile timely + +//! differential-dataflow + the generated engine, ~tens of seconds; the +//! binary is then reused). Stale cache entries are pruned so runtime-compile +//! artifacts don't accumulate unbounded (DISK constraint). +//! 2. `DriverHandle::spawn()` — launch the built binary ONCE as a child with +//! piped stdin/stdout. The flowlog engine inside stays warm across every +//! `commit`, preserving incrementality. +//! 3. `insert` / `remove` / `commit` — speak the protocol over the pipe; one +//! `commit` returns this epoch's output deltas. +//! +//! ## Cache location +//! +//! `$EGGLOG_FLOWLOG_CACHE` if set, else `/egglog-flowlog-cache`. +//! All runtime-compile target dirs live UNDER this single root. + +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; + +use anyhow::{anyhow, Context, Result}; + +use crate::codegen; + +/// Absolute path to the local flowlog-rs clone. The runtime driver crate +/// depends on `flowlog-runtime` / `flowlog-build` by path from here. +const FLOWLOG_ROOT: &str = "/tmp/flowlog-main"; + +/// Keep at most this many cached driver builds; older ones are pruned to bound +/// disk use from runtime compilation (DISK constraint in the brief). +const MAX_CACHE_ENTRIES: usize = 8; + +/// FNV-1a 64-bit hash of the `.dl` text — the rule-set cache key. A given +/// rule-set hashes identically every run, so it compiles once and is reused. +fn hash_dl(dl: &str) -> String { + let mut h: u64 = 0xcbf29ce484222325; + for b in dl.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + format!("{h:016x}") +} + +/// The single cache root for all runtime-compile artifacts. +fn cache_root() -> PathBuf { + if let Some(dir) = std::env::var_os("EGGLOG_FLOWLOG_CACHE") { + PathBuf::from(dir) + } else { + std::env::temp_dir().join("egglog-flowlog-cache") + } +} + +/// Prune all but the most-recently-modified `MAX_CACHE_ENTRIES` cache dirs, +/// never removing `keep`. Best-effort; failures are ignored. +fn prune_cache(root: &Path, keep: &Path) { + let Ok(rd) = std::fs::read_dir(root) else { + return; + }; + let mut entries: Vec<(std::time::SystemTime, PathBuf)> = rd + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir() && e.path() != keep) + .filter_map(|e| { + let mtime = e.metadata().and_then(|m| m.modified()).ok()?; + Some((mtime, e.path())) + }) + .collect(); + if entries.len() < MAX_CACHE_ENTRIES { + return; + } + entries.sort_by_key(|(t, _)| *t); + let excess = entries.len() + 1 - MAX_CACHE_ENTRIES; + for (_, path) in entries.into_iter().take(excess) { + let _ = std::fs::remove_dir_all(&path); + } +} + +/// A built, possibly-running driver subprocess. +pub struct DriverHandle { + /// Path to the compiled driver binary (cached by rule-set hash). + binary: PathBuf, + /// The spawned child + its piped stdio, once `spawn()` is called. + child: Option, + stdin: Option, + stdout: Option>, +} + +impl DriverHandle { + /// Build the driver for `dl` (or reuse a cached binary keyed by its hash), + /// returning a not-yet-spawned handle. This is where the (cold) `cargo + /// build` happens. + pub fn build_or_cached(dl: &str) -> Result { + let root = cache_root(); + std::fs::create_dir_all(&root) + .with_context(|| format!("creating cache root {}", root.display()))?; + + let hash = hash_dl(dl); + let crate_dir = root.join(&hash); + // The release binary name matches the crate name (`driver`). + let binary = crate_dir.join("target").join("release").join("driver"); + + if binary.exists() { + // Cache hit: touch the dir so LRU pruning keeps it warm. + let _ = std::fs::write(crate_dir.join(".touch"), hash.as_bytes()); + prune_cache(&root, &crate_dir); + return Ok(DriverHandle { + binary, + child: None, + stdin: None, + stdout: None, + }); + } + + // Cache miss: materialize the temp crate and shell out to cargo build. + Self::materialize_crate(&crate_dir, dl)?; + Self::cargo_build(&crate_dir)?; + + if !binary.exists() { + return Err(anyhow!( + "driver build reported success but binary {} is missing", + binary.display() + )); + } + prune_cache(&root, &crate_dir); + Ok(DriverHandle { + binary, + child: None, + stdin: None, + stdout: None, + }) + } + + /// Build the driver for a generalized join: a `.dl` and a matching driver + /// `main.rs` are supplied directly (the dd-join path's flexible-arity, + /// multi-relation protocol), keyed by the hash of `dl ++ main_rs` so each + /// distinct join shape compiles once and is reused. + pub fn build_or_cached_with(dl: &str, main_rs: &str) -> Result { + let root = cache_root(); + std::fs::create_dir_all(&root) + .with_context(|| format!("creating cache root {}", root.display()))?; + + let combined = format!("{dl}\n//---main---\n{main_rs}"); + let hash = hash_dl(&combined); + let crate_dir = root.join(&hash); + let binary = crate_dir.join("target").join("release").join("driver"); + + if binary.exists() { + let _ = std::fs::write(crate_dir.join(".touch"), hash.as_bytes()); + prune_cache(&root, &crate_dir); + return Ok(DriverHandle { + binary, + child: None, + stdin: None, + stdout: None, + }); + } + + Self::materialize_crate_with(&crate_dir, dl, main_rs)?; + Self::cargo_build(&crate_dir)?; + if !binary.exists() { + return Err(anyhow!( + "driver build reported success but binary {} is missing", + binary.display() + )); + } + prune_cache(&root, &crate_dir); + Ok(DriverHandle { + binary, + child: None, + stdin: None, + stdout: None, + }) + } + + /// Write the driver crate's files to `crate_dir`. + fn materialize_crate(crate_dir: &Path, dl: &str) -> Result<()> { + Self::materialize_crate_with(crate_dir, dl, &codegen::emit_main_rs()) + } + + /// Write the driver crate's files with an explicit `main.rs`. + fn materialize_crate_with(crate_dir: &Path, dl: &str, main_rs: &str) -> Result<()> { + let src = crate_dir.join("src"); + std::fs::create_dir_all(&src).with_context(|| format!("creating {}", src.display()))?; + std::fs::write( + crate_dir.join("Cargo.toml"), + codegen::emit_cargo_toml("driver", FLOWLOG_ROOT), + )?; + std::fs::write(crate_dir.join("build.rs"), codegen::emit_build_rs())?; + std::fs::write(crate_dir.join("program.dl"), dl)?; + std::fs::write(src.join("main.rs"), main_rs)?; + Ok(()) + } + + /// Shell out to `cargo build --release` in the temp crate dir. + fn cargo_build(crate_dir: &Path) -> Result<()> { + let output = Command::new("cargo") + .arg("build") + .arg("--release") + .current_dir(crate_dir) + // Confine all build artifacts to this crate's own target dir + // (under the single cache root). + .env("CARGO_TARGET_DIR", crate_dir.join("target")) + .output() + .with_context(|| format!("spawning cargo build in {}", crate_dir.display()))?; + if !output.status.success() { + return Err(anyhow!( + "cargo build failed for driver in {}:\n--- stdout ---\n{}\n--- stderr ---\n{}", + crate_dir.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + Ok(()) + } + + /// Spawn the driver binary once, wiring up the stdin/stdout pipe. The + /// flowlog engine inside stays warm for the whole program. + pub fn spawn(&mut self) -> Result<()> { + if self.child.is_some() { + return Ok(()); + } + let mut child = Command::new(&self.binary) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .with_context(|| format!("spawning driver {}", self.binary.display()))?; + let stdin = child.stdin.take().ok_or_else(|| anyhow!("no stdin"))?; + let stdout = child.stdout.take().ok_or_else(|| anyhow!("no stdout"))?; + self.stdin = Some(stdin); + self.stdout = Some(BufReader::new(stdout)); + self.child = Some(child); + Ok(()) + } + + fn send(&mut self, line: &str) -> Result<()> { + let stdin = self + .stdin + .as_mut() + .ok_or_else(|| anyhow!("driver not spawned"))?; + stdin.write_all(line.as_bytes())?; + stdin.write_all(b"\n")?; + stdin.flush()?; + Ok(()) + } + + /// Stage an `insert ` delta. + pub fn insert(&mut self, rel: &str, a: i32, b: i32) -> Result<()> { + self.send(&format!("insert {rel} {a} {b}")) + } + + /// Stage a `remove ` delta (M2 protocol; retraction wiring is + /// M3, but the command is supported end-to-end). + #[allow(dead_code)] + pub fn remove(&mut self, rel: &str, a: i32, b: i32) -> Result<()> { + self.send(&format!("remove {rel} {a} {b}")) + } + + /// Send `clear` and wait for the `ok` reply: reset the dd-join engine's + /// staged relations to empty before re-staging the next iteration's read + /// view (the non-recursive join is recomputed from scratch each call). + pub fn send_clear(&mut self) -> Result<()> { + self.send("clear")?; + let stdout = self + .stdout + .as_mut() + .ok_or_else(|| anyhow!("driver not spawned"))?; + let mut buf = String::new(); + loop { + buf.clear(); + let n = stdout.read_line(&mut buf)?; + if n == 0 { + return Err(anyhow!("driver closed pipe before `ok` (clear)")); + } + let line = buf.trim(); + if line == "ok" { + break; + } + if let Some(err) = line.strip_prefix("err ") { + return Err(anyhow!("driver error: {err}")); + } + } + Ok(()) + } + + /// Stage an `ins ...` delta for the generalized + /// dd-join protocol (arbitrary-arity row into relation `rel_idx`). + pub fn insert_row(&mut self, rel_idx: usize, cols: &[i32]) -> Result<()> { + let mut line = format!("ins {rel_idx}"); + for c in cols { + line.push(' '); + line.push_str(&c.to_string()); + } + self.send(&line) + } + + /// `commit` one epoch for the generalized dd-join protocol; read back + /// `row ...` lines (one per join binding) until the terminating + /// `ok`. Returns the binding rows as `Vec>`. + pub fn commit_rows(&mut self) -> Result>> { + self.send("commit")?; + let stdout = self + .stdout + .as_mut() + .ok_or_else(|| anyhow!("driver not spawned"))?; + let mut rows = Vec::new(); + let mut buf = String::new(); + loop { + buf.clear(); + let n = stdout.read_line(&mut buf)?; + if n == 0 { + return Err(anyhow!("driver closed pipe before `ok`")); + } + let line = buf.trim(); + if line == "ok" { + break; + } + if let Some(rest) = line.strip_prefix("row ") { + let cols: Vec = rest + .split_whitespace() + .filter_map(|t| t.parse::().ok()) + .collect(); + rows.push(cols); + } else if let Some(err) = line.strip_prefix("err ") { + return Err(anyhow!("driver error: {err}")); + } + } + Ok(rows) + } + + /// `commit` one epoch; read back `delta ` lines until + /// the terminating `ok`. Returns the `hop` deltas as `(x, z, diff)`. + pub fn commit(&mut self) -> Result> { + self.send("commit")?; + let stdout = self + .stdout + .as_mut() + .ok_or_else(|| anyhow!("driver not spawned"))?; + let mut deltas = Vec::new(); + let mut buf = String::new(); + loop { + buf.clear(); + let n = stdout.read_line(&mut buf)?; + if n == 0 { + return Err(anyhow!("driver closed pipe before `ok`")); + } + let line = buf.trim(); + if line == "ok" { + break; + } + if let Some(rest) = line.strip_prefix("delta ") { + let toks: Vec<&str> = rest.split_whitespace().collect(); + // `hop ` + if toks.len() == 4 { + let x: i32 = toks[1].parse().context("delta x")?; + let z: i32 = toks[2].parse().context("delta z")?; + let d: i32 = toks[3].parse().context("delta diff")?; + deltas.push((x, z, d)); + } + } else if let Some(err) = line.strip_prefix("err ") { + return Err(anyhow!("driver error: {err}")); + } + } + Ok(deltas) + } +} + +impl Drop for DriverHandle { + fn drop(&mut self) { + // Ask the driver to quit, then reap it so we don't leak timely worker + // threads / zombie processes across egraphs. + let _ = self.send("quit"); + if let Some(mut child) = self.child.take() { + let _ = child.wait(); + } + } +} diff --git a/egglog-experimental/flowlog/tests/corpus.rs b/egglog-experimental/flowlog/tests/corpus.rs new file mode 100644 index 0000000..ed3dcf3 --- /dev/null +++ b/egglog-experimental/flowlog/tests/corpus.rs @@ -0,0 +1,65 @@ +//! Differential test: run a curated subset of egglog's own `.egg` corpus on +//! BOTH the reference in-memory backend and the FlowLog backend, and assert +//! they agree — unconditionally (no env gating), as part of `cargo test`. +//! +//! Both egraphs are built identically (`EGraph::default()` vs +//! `EGraph::with_backend(FlowlogEGraph)`), so the frontend drives the exact +//! same operations into each backend. We append `(print-size)` to every file +//! and compare its output (per-function row counts) between the two backends: +//! if FlowLog's differential-dataflow engine derives a different final +//! database than the reference, the counts diverge and the test fails. +//! +//! FlowLog implements only a subset of egglog (no proofs / containers / +//! subsumption / complex merges, and every rule body must be DD-lowerable), so +//! the list below is limited to corpus files that run on it. + +use egglog::EGraph; + +/// egglog corpus files that run on FlowLog and agree with the reference backend. +const CORPUS: &[&str] = &[ + "bool.egg", + "i64.egg", + "bitwise.egg", + "interval.egg", + "before-proofs.egg", + "merge-saturates.egg", + "string.egg", +]; + +/// Run `program` and return the output of the trailing `(print-size)` as text. +fn print_size_of(mut eg: EGraph, file: &str, program: &str) -> String { + let outputs = eg + .parse_and_run_program(Some(file.to_string()), program) + .unwrap_or_else(|e| panic!("run failed on {file}: {e}")); + // The appended `(print-size)` is the last command; its output summarizes + // every function's row count deterministically. + outputs.last().map(|o| o.to_string()).unwrap_or_default() +} + +#[test] +fn flowlog_matches_reference_backend() { + for file in CORPUS { + let path = std::path::Path::new("../../egglog/tests").join(file); + let src = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + // Append `(print-size)` so the trailing output is a full per-function + // row-count summary of the final database. + let program = format!("{src}\n(print-size)"); + + let reference = print_size_of(EGraph::default(), file, &program); + let flowlog = print_size_of( + EGraph::with_backend(Box::new( + egglog_experimental_flowlog::EGraph::new_interpret(), + )), + file, + &program, + ); + + assert_eq!( + reference, flowlog, + "FlowLog and the reference backend disagree on {file}:\n\ + --- reference (print-size) ---\n{reference}\n\ + --- flowlog (print-size) ---\n{flowlog}", + ); + } +} diff --git a/egglog-experimental/flowlog/tests/rewrite_join.rs b/egglog-experimental/flowlog/tests/rewrite_join.rs new file mode 100644 index 0000000..d558be3 --- /dev/null +++ b/egglog-experimental/flowlog/tests/rewrite_join.rs @@ -0,0 +1,48 @@ +// Confirms that a real rewrite rule fires through the FlowLog differential- +// dataflow join in Interpret mode: without the rule body join producing a +// match and the head applying it (then congruence closing), the final +// `(check ...)` would fail. +use egglog::EGraph; + +fn flowlog_egraph() -> EGraph { + EGraph::with_backend(Box::new( + egglog_experimental_flowlog::EGraph::new_interpret(), + )) +} + +#[test] +fn commutativity_fires_through_dd_join() { + let mut eg = flowlog_egraph(); + eg.parse_and_run_program( + None, + r#" + (datatype Math (Num i64) (Add Math Math)) + (rewrite (Add a b) (Add b a)) + (let t (Add (Num 1) (Num 2))) + (run 3) + ; only holds if (Add a b) matched on the DD join, the head built + ; (Add b a), and congruence unified the two: + (check (= (Add (Num 1) (Num 2)) (Add (Num 2) (Num 1)))) + "#, + ) + .expect("commutativity rewrite should fire and unify via the DD join"); +} + +#[test] +fn multi_atom_join_associativity() { + // A two-atom body join ((Add a b) matched against the rewrite LHS decomposed + // into view atoms) exercising a wider DD join than a single atom. + let mut eg = flowlog_egraph(); + eg.parse_and_run_program( + None, + r#" + (datatype Math (Num i64) (Add Math Math) (Mul Math Math)) + (rewrite (Mul a (Add b c)) (Add (Mul a b) (Mul a c))) + (let e (Mul (Num 2) (Add (Num 3) (Num 4)))) + (run 3) + (check (= (Mul (Num 2) (Add (Num 3) (Num 4))) + (Add (Mul (Num 2) (Num 3)) (Mul (Num 2) (Num 4))))) + "#, + ) + .expect("distributivity rewrite should fire via the multi-atom DD join"); +} diff --git a/egglog-experimental/flowlog/tests/smoke.rs b/egglog-experimental/flowlog/tests/smoke.rs new file mode 100644 index 0000000..026c9b4 --- /dev/null +++ b/egglog-experimental/flowlog/tests/smoke.rs @@ -0,0 +1,12 @@ +use egglog::EGraph; + +#[test] +fn flowlog_runs_basic_egg() { + let backend = Box::new(egglog_experimental_flowlog::EGraph::new_interpret()); + let mut eg = EGraph::with_backend(backend); + eg.parse_and_run_program( + None, + "(datatype Math (Num i64) (Add Math Math))\n(Add (Num 1) (Num 2))\n(run 1)\n(print-size Add)", + ) + .unwrap(); +} diff --git a/egglog-experimental/flowlog/transitive_step.dl b/egglog-experimental/flowlog/transitive_step.dl new file mode 100644 index 0000000..9d5a19f --- /dev/null +++ b/egglog-experimental/flowlog/transitive_step.dl @@ -0,0 +1,35 @@ +// Milestone-1 build-time-FIXED FlowLog program: ONE transitive-closure hop. +// +// This `.dl` is deliberately NON-RECURSIVE. egglog's `(run N)` applies a +// ruleset N times *with bounded extension per round* — a transitive-closure +// rule must extend N hops, NOT saturate to the full closure in one step. So we +// compile a single non-recursive join: +// +// hop(x, z) :- path(x, y), edge(y, z). +// +// Both `path` and `edge` are runtime-staged command inputs (IO="command", the +// attribute that gives the generated incremental engine its `insert_` / +// `remove_` staging methods). `hop` is the output relation: its per-epoch +// deltas come back in `IncrementalResults.hop`. +// +// The host backend (lib.rs `run_rules`) realizes egglog's bounded N-round loop: +// one `run_rules` call = one `commit()` = one hop. Each round it feeds the +// PREVIOUS round's newly-derived `path` rows as an `insert_path` delta; the +// single non-recursive join then produces exactly the next hop's `hop` deltas, +// which the host folds back into the `path` mirror. N calls = N hops, bounded +// (NOT closure). This mirrors the proven Feldera M1 host-feedback model, but on +// real flowlog-rs differential dataflow. +// +// Relation names are lowercase so the generated public API idents are +// predictable (insert_path / insert_edge / IncrementalResults.hop) — flowlog +// normalizes relation names to lowercase for the lib API (SPIKE_RESULTS.md). + +.decl edge(src: int32, dst: int32) +.input edge(IO="command", delimiter=",") + +.decl path(src: int32, dst: int32) +.input path(IO="command", delimiter=",") + +.decl hop(src: int32, dst: int32) +hop(x, z) :- path(x, y), edge(y, z). +.output hop diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/Cargo.toml b/egglog-experimental/flowlog/vendor/differential-dogs3/Cargo.toml new file mode 100644 index 0000000..32b0af4 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/Cargo.toml @@ -0,0 +1,38 @@ +[package] +edition = "2021" +rust-version = "1.86" +name = "differential-dogs3" +# Vendored from crates.io `differential-dogs3 = 0.24.0` (Frank McSherry's +# `dogsdogsdogs`: worst-case-optimal delta-query operators for differential +# dataflow) with ONE bug fix: `src/operators/lookup_map.rs` read +# `key_con.index(1)` of a length-1 key container, panicking on EVERY lookup +# against its own pinned differential-dataflow 0.24.0 (`index out of bounds: +# the len is 1 but the index is 1`). Both occurrences changed to +# `key_con.index(0)`. This is an upstreamable bug fix, not a workaround. +# Used by `egglog-bridge-flowlog`'s `--wcoj` triangle WCOJ (`dd_native.rs`). +version = "0.24.0" +authors = ["Frank McSherry "] +description = "Advanced join patterns in differential dataflow (vendored + patched)" +homepage = "https://github.com/TimelyDataflow/differential-dataflow" +license = "MIT" +repository = "https://github.com/TimelyDataflow/differential-dataflow.git" +publish = false + +[features] +default = [] + +[lib] +name = "differential_dogs3" +path = "src/lib.rs" + +[dependencies.differential-dataflow] +version = "=0.24.0" +default-features = false + +[dependencies.serde] +version = "1.0" +features = ["derive"] + +[dependencies.timely] +version = "=0.30.0" +default-features = false diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/altneu.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/altneu.rs new file mode 100644 index 0000000..0ddb957 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/altneu.rs @@ -0,0 +1,100 @@ +//! A lexicographically ordered pair of timestamps. +//! +//! Two timestamps (s1, t1) and (s2, t2) are ordered either if +//! s1 and s2 are ordered, or if s1 equals s2 and t1 and t2 are +//! ordered. +//! +//! The join of two timestamps should have as its first coordinate +//! the join of the first coordinates, and for its second coordinate +//! the join of the second coordinates for elements whose first +//! coordinate equals the computed join. That may be the minimum +//! element of the second lattice, if neither first element equals +//! the join. + +use serde::{Deserialize, Serialize}; + +/// A pair of timestamps, partially ordered by the product order. +#[derive(Debug, Hash, Default, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct AltNeu { + pub time: T, + pub neu: bool, // alt < neu in timestamp comparisons. +} + +impl AltNeu { + pub fn alt(time: T) -> Self { AltNeu { time, neu: false } } + pub fn neu(time: T) -> Self { AltNeu { time, neu: true } } +} + +// Implement timely dataflow's `PartialOrder` trait. +use timely::order::PartialOrder; +impl PartialOrder for AltNeu { + fn less_equal(&self, other: &Self) -> bool { + if self.time.eq(&other.time) { + self.neu <= other.neu + } + else { + self.time.less_equal(&other.time) + } + } +} + +// Implement timely dataflow's `PathSummary` trait. +// This is preparation for the `Timestamp` implementation below. +use timely::progress::PathSummary; +impl PathSummary> for () { + fn results_in(&self, timestamp: &AltNeu) -> Option> { + Some(timestamp.clone()) + } + fn followed_by(&self, other: &Self) -> Option { + Some(other.clone()) + } +} + +// Implement timely dataflow's `Timestamp` trait. +use timely::progress::Timestamp; +impl Timestamp for AltNeu { + type Summary = (); + fn minimum() -> Self { AltNeu::alt(T::minimum()) } +} + +use timely::progress::timestamp::Refines; + +impl Refines for AltNeu { + fn to_inner(other: T) -> Self { + AltNeu::alt(other) + } + fn to_outer(self: AltNeu) -> T { + self.time + } + fn summarize(_path: ()) -> T::Summary { + Default::default() + } +} + +// Implement differential dataflow's `Lattice` trait. +// This extends the `PartialOrder` implementation with additional structure. +use differential_dataflow::lattice::Lattice; +impl Lattice for AltNeu { + fn join(&self, other: &Self) -> Self { + let time = self.time.join(&other.time); + let mut neu = false; + if time == self.time { + neu = neu || self.neu; + } + if time == other.time { + neu = neu || other.neu; + } + AltNeu { time, neu } + } + fn meet(&self, other: &Self) -> Self { + let time = self.time.meet(&other.time); + let mut neu = true; + if time == self.time { + neu = neu && self.neu; + } + if time == other.time { + neu = neu && other.neu; + } + AltNeu { time, neu } + } +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/calculus.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/calculus.rs new file mode 100644 index 0000000..f7697e1 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/calculus.rs @@ -0,0 +1,67 @@ +//! Traits and implementations for differentiating and integrating collections. +//! +//! The `Differentiate` and `Integrate` traits allow us to move between standard differential +//! dataflow collections, and collections that describe their instantaneous change. The first +//! trait converts a collection to one that contains each change at the moment it occurs, but +//! then immediately retracting it. The second trait takes such a representation are recreates +//! the collection from its instantaneous changes. +//! +//! These two traits together allow us to build dataflows that maintain computates over inputs +//! that are the instantaneous changes, and then to reconstruct collections from them. The most +//! clear use case for this are "delta query" implementations of relational joins, where linearity +//! allows us to write dataflows based on instantaneous changes, whose "accumluated state" is +//! almost everywhere empty (and so has a low memory footprint, if the system works as planned). + +use timely::dataflow::Scope; +use timely::progress::Timestamp; +use timely::dataflow::operators::vec::{Filter, Map}; +use differential_dataflow::{AsCollection, VecCollection, Data}; +use differential_dataflow::difference::Abelian; + +use crate::altneu::AltNeu; + +/// Produce a collection containing the changes at the moments they happen. +pub trait Differentiate<'scope, T: Timestamp, D: Data, R: Abelian> { + fn differentiate<'inner>(self, child: Scope<'inner, AltNeu>) -> VecCollection<'inner, AltNeu, D, R>; +} + +/// Collect instantaneous changes back in to a collection. +pub trait Integrate<'scope, T: Timestamp, D: Data, R: Abelian> { + fn integrate<'outer>(self, outer: Scope<'outer, T>) -> VecCollection<'outer, T, D, R>; +} + +impl<'scope, T, D, R> Differentiate<'scope, T, D, R> for VecCollection<'scope, T, D, R> +where + T: Timestamp, + D: Data, + R: Abelian + 'static, +{ + // For each (data, Alt(time), diff) we add a (data, Neu(time), -diff). + fn differentiate<'inner>(self, child: Scope<'inner, AltNeu>) -> VecCollection<'inner, AltNeu, D, R> { + self.enter(child) + .inner + .flat_map(|(data, time, diff)| { + let mut neg_diff = diff.clone(); + neg_diff.negate(); + let neu = (data.clone(), AltNeu::neu(time.time.clone()), neg_diff); + let alt = (data, time, diff); + Some(alt).into_iter().chain(Some(neu)) + }) + .as_collection() + } +} + +impl<'scope, T, D, R> Integrate<'scope, T, D, R> for VecCollection<'scope, AltNeu, D, R> +where + T: Timestamp, + D: Data, + R: Abelian + 'static, +{ + // We discard each `neu` variant and strip off the `alt` wrapper. + fn integrate<'outer>(self, outer: Scope<'outer, T>) -> VecCollection<'outer, T, D, R> { + self.inner + .filter(|(_d,t,_r)| !t.neu) + .as_collection() + .leave(outer) + } +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/lib.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/lib.rs new file mode 100644 index 0000000..6480fc6 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/lib.rs @@ -0,0 +1,213 @@ +// Vendored + one-line-patched third-party crate (see Cargo.toml). Silence +// clippy on the upstream source — it is not ours to restyle. +#![allow(clippy::all)] +#![allow(clippy::pedantic)] + +use std::hash::Hash; + +use timely::progress::Timestamp; +use timely::dataflow::operators::vec::Partition; +use timely::dataflow::operators::Concatenate; + +use differential_dataflow::{ExchangeData, VecCollection, AsCollection}; +use differential_dataflow::difference::{Monoid, Multiply}; +use differential_dataflow::lattice::Lattice; +use differential_dataflow::operators::arrange::TraceAgent; + +pub mod altneu; +pub mod calculus; +pub mod operators; + +/// A type capable of extending a stream of prefixes. +/// +/** + Implementors of `PrefixExtension` provide types and methods for extending a differential dataflow collection, + via the three methods `count`, `propose`, and `validate`. +**/ +pub trait PrefixExtender<'scope, T: Timestamp, R: Monoid+Multiply> { + /// The required type of prefix to extend. + type Prefix; + /// The type to be produced as extension. + type Extension; + /// Annotates prefixes with the number of extensions the relation would propose. + fn count(&mut self, prefixes: VecCollection<'scope, T, (Self::Prefix, usize, usize), R>, index: usize) -> VecCollection<'scope, T, (Self::Prefix, usize, usize), R>; + /// Extends each prefix with corresponding extensions. + fn propose(&mut self, prefixes: VecCollection<'scope, T, Self::Prefix, R>) -> VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>; + /// Restricts proposed extensions by those the extender would have proposed. + fn validate(&mut self, extensions: VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>) -> VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>; +} + +pub trait ProposeExtensionMethod<'scope, T: Timestamp, P: ExchangeData+Ord, R: Monoid+Multiply> { + fn propose_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, PE::Extension), R>; + fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T,R,Prefix=P,Extension=E>]) -> VecCollection<'scope, T, (P, E), R>; +} + +impl<'scope, T, P, R> ProposeExtensionMethod<'scope, T, P, R> for VecCollection<'scope, T, P, R> +where + T: Timestamp, + P: ExchangeData+Ord, + R: Monoid+Multiply+'static, +{ + fn propose_using(self, extender: &mut PE) -> VecCollection<'scope, T, (P, PE::Extension), R> + where + PE: PrefixExtender<'scope, T, R, Prefix=P> + { + extender.propose(self) + } + fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T,R,Prefix=P,Extension=E>]) -> VecCollection<'scope, T, (P, E), R> + where + E: ExchangeData+Ord + { + + if extenders.len() == 1 { + extenders[0].propose(self) + } + else { + let mut counts = self.clone().map(|p| (p, 1 << 31, 0)); + for (index,extender) in extenders.iter_mut().enumerate() { + counts = extender.count(counts, index); + } + + let parts = counts.inner.partition(extenders.len() as u64, |((p, _, i),t,d)| (i as u64, (p,t,d))); + + let mut results = Vec::new(); + for (index, nominations) in parts.into_iter().enumerate() { + let mut extensions = extenders[index].propose(nominations.as_collection()); + for other in (0..extenders.len()).filter(|&x| x != index) { + extensions = extenders[other].validate(extensions); + } + + results.push(extensions.inner); // save extensions + } + + self.scope().concatenate(results).as_collection() + } + } +} + +pub trait ValidateExtensionMethod<'scope, T: Timestamp, R: Monoid+Multiply, P, E> { + fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, E), R>; +} + +impl<'scope, T: Timestamp, R: Monoid+Multiply, P, E> ValidateExtensionMethod<'scope, T, R, P, E> for VecCollection<'scope, T, (P, E), R> { + fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, E), R> { + extender.validate(self) + } +} + +// These are all defined here so that users can be assured a common layout. +use differential_dataflow::trace::implementations::{KeySpine, ValSpine}; +type TraceValHandle = TraceAgent>; +type TraceKeyHandle = TraceAgent>; + +pub struct CollectionIndex +where + K: ExchangeData, + V: ExchangeData, + T: Lattice+ExchangeData+Timestamp, + R: Monoid+Multiply+ExchangeData, +{ + /// A trace of type (K, ()), used to count extensions for each prefix. + count_trace: TraceKeyHandle, + + /// A trace of type (K, V), used to propose extensions for each prefix. + propose_trace: TraceValHandle, + + /// A trace of type ((K, V), ()), used to validate proposed extensions. + validate_trace: TraceKeyHandle<(K, V), T, R>, +} + +impl Clone for CollectionIndex +where + K: ExchangeData+Hash, + V: ExchangeData+Hash, + T: Lattice+ExchangeData+Timestamp, + R: Monoid+Multiply+ExchangeData, +{ + fn clone(&self) -> Self { + CollectionIndex { + count_trace: self.count_trace.clone(), + propose_trace: self.propose_trace.clone(), + validate_trace: self.validate_trace.clone(), + } + } +} + +impl CollectionIndex +where + K: ExchangeData+Hash, + V: ExchangeData+Hash, + T: Lattice+ExchangeData+Timestamp, + R: Monoid+Multiply+ExchangeData, +{ + + pub fn index<'scope>(collection: VecCollection<'scope, T, (K, V), R>) -> Self { + // We need to count the number of (k, v) pairs and not rely on the given Monoid R and its binary addition operation. + // counts and validate can share the base arrangement + let arranged = collection.clone().arrange_by_self(); + // TODO: This could/should be arrangement to arrangement, via `reduce_abelian`, but the types are a mouthful at the moment. + let counts = arranged + .clone() + .as_collection(|k,_v| k.clone()) + .distinct() + .map(|(k, _v)| k) + .arrange_by_self() + .trace; + let propose = collection.arrange_by_key().trace; + let validate = arranged.trace; + + CollectionIndex { + count_trace: counts, + propose_trace: propose, + validate_trace: validate, + } + } + pub fn extend_usingK+Clone>(&self, logic: F) -> CollectionExtender { + CollectionExtender { + phantom: std::marker::PhantomData, + indices: self.clone(), + key_selector: logic, + } + } +} + +pub struct CollectionExtender +where + K: ExchangeData, + V: ExchangeData, + T: Lattice+ExchangeData+Timestamp, + R: Monoid+Multiply+ExchangeData, + F: Fn(&P)->K+Clone, +{ + phantom: std::marker::PhantomData

, + indices: CollectionIndex, + key_selector: F, +} + +impl<'scope, T, K, V, R, P, F> PrefixExtender<'scope, T, R> for CollectionExtender +where + T: Timestamp + Lattice + ExchangeData + Hash, + K: ExchangeData+Hash+Default, + V: ExchangeData+Hash+Default, + P: ExchangeData, + R: Monoid+Multiply+ExchangeData, + F: Fn(&P)->K+Clone+'static, +{ + type Prefix = P; + type Extension = V; + + fn count(&mut self, prefixes: VecCollection<'scope, T, (P, usize, usize), R>, index: usize) -> VecCollection<'scope, T, (P, usize, usize), R> { + let counts = self.indices.count_trace.import(prefixes.scope()); + operators::count::count(prefixes, counts, self.key_selector.clone(), index) + } + + fn propose(&mut self, prefixes: VecCollection<'scope, T, P, R>) -> VecCollection<'scope, T, (P, V), R> { + let propose = self.indices.propose_trace.import(prefixes.scope()); + operators::propose::propose(prefixes, propose, self.key_selector.clone()) + } + + fn validate(&mut self, extensions: VecCollection<'scope, T, (P, V), R>) -> VecCollection<'scope, T, (P, V), R> { + let validate = self.indices.validate_trace.import(extensions.scope()); + operators::validate::validate(extensions, validate, self.key_selector.clone()) + } +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/count.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/count.rs new file mode 100644 index 0000000..34aebc7 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/count.rs @@ -0,0 +1,40 @@ +use differential_dataflow::{ExchangeData, VecCollection, Hashable}; +use differential_dataflow::difference::{Semigroup, Monoid, Multiply}; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::TraceReader; + +/// Reports a number of extensions to a stream of prefixes. +/// +/// This method takes as input a stream of `(prefix, count, index)` triples. +/// For each triple, it extracts a key using `key_selector`, and finds the +/// associated count in `arrangement`. If the found count is less than `count`, +/// the `count` and `index` fields are overwritten with their new values. +pub fn count<'scope, Tr, K, R, F, P>( + prefixes: VecCollection<'scope, Tr::Time, (P, usize, usize), R>, + arrangement: Arranged<'scope, Tr>, + key_selector: F, + index: usize, +) -> VecCollection<'scope, Tr::Time, (P, usize, usize), R> +where + Tr: TraceReader+Clone+'static, + Tr::KeyContainer: differential_dataflow::trace::implementations::BatchContainer, + for<'a> Tr::Diff : Semigroup>, + K: Hashable + Ord + Default + 'static, + R: Monoid+Multiply+ExchangeData, + F: Fn(&P)->K+Clone+'static, + P: ExchangeData, +{ + crate::operators::lookup_map( + prefixes, + arrangement, + move |p: &(P,usize,usize), k: &mut K| { *k = key_selector(&p.0); }, + move |(p,c,i), r, _, s| { + let s = *s as usize; + if *c < s { ((p.clone(), *c, *i), r.clone()) } + else { ((p.clone(), s, index), r.clone()) } + }, + Default::default(), + Default::default(), + Default::default(), + ) +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join.rs new file mode 100644 index 0000000..3683635 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join.rs @@ -0,0 +1,368 @@ +//! Dataflow operator for delta joins over partially ordered timestamps. +//! +//! Given multiple streams of updates `(data, time, diff)` that are each +//! defined over the same partially ordered `time`, we want to form the +//! full cross-join of all relations (we will *later* apply some filters +//! and instead equijoin on keys). +//! +//! The "correct" output is the outer join of these triples, where +//! 1. The `data` entries are just tuple'd up together, +//! 2. The `time` entries are subjected to the lattice `join` operator, +//! 3. The `diff` entries are multiplied. +//! +//! One way to produce the correct output is to form independent dataflow +//! fragments for each input stream, such that each intended output is then +//! produced by exactly one of these input streams. +//! +//! There are several incorrect ways one might do this, but here is one way +//! that I hope is not incorrect: +//! +//! Each input stream of updates is joined with each other input collection, +//! where each input update is matched against each other input update that +//! has a `time` that is less-than the input update's `time`, *UNDER A TOTAL +//! ORDER ON `time`*. The output are the `(data, time, diff)` entries that +//! follow the rules above, except that we additionally preserve the input's +//! initial `time` as well, for use in subsequent joins with the other input +//! collections. +//! +//! There are some caveats about ties, and we should treat each `time` for +//! each input as occurring at distinct times, one after the other (so that +//! ties are resolved by the index of the input). There is also the matter +//! of logical compaction, which should not be done in a way that prevents +//! the correct determination of the total order comparison. + +use std::collections::HashMap; +use std::ops::Mul; +use std::time::Instant; + +use timely::ContainerBuilder; +use timely::container::CapacityContainerBuilder; +use timely::dataflow::Stream; +use timely::dataflow::channels::pact::{Pipeline, Exchange}; +use timely::dataflow::operators::{Capability, Operator, generic::Session}; +use timely::PartialOrder; +use timely::progress::Antichain; +use timely::progress::frontier::AntichainRef; + +use differential_dataflow::{ExchangeData, VecCollection, AsCollection, Hashable}; +use differential_dataflow::difference::{Monoid, Semigroup}; +use differential_dataflow::lattice::Lattice; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::{Cursor, TraceReader}; +use differential_dataflow::consolidation::{consolidate, consolidate_updates}; +use differential_dataflow::trace::implementations::BatchContainer; + +/// A binary equijoin that responds to updates on only its first input. +/// +/// This operator responds to inputs of the form +/// +/// ```ignore +/// ((key, val1, time1), initial_time, diff1) +/// ``` +/// +/// where `initial_time` is less or equal to `time1`, and produces as output +/// +/// ```ignore +/// ((output_func(key, val1, val2), lub(time1, time2)), initial_time, diff1 * diff2) +/// ``` +/// +/// for each `((key, val2), time2, diff2)` present in `arrangement`, where +/// `time2` is less than `initial_time` *UNDER THE TOTAL ORDER ON TIMES*. +/// This last constraint is important to ensure that we correctly produce +/// all pairs of output updates across multiple `half_join` operators. +/// +/// Notice that the time is hoisted up into data. The expectation is that +/// once out of the "delta flow region", the updates will be `delay`d to the +/// times specified in the payloads. +pub fn half_join<'scope, K, V, R, Tr, FF, CF, DOut, S>( + stream: VecCollection<'scope, Tr::Time, (K, V, Tr::Time), R>, + arrangement: Arranged<'scope, Tr>, + frontier_func: FF, + comparison: CF, + mut output_func: S, +) -> VecCollection<'scope, Tr::Time, (DOut, Tr::Time), >::Output> +where + K: Hashable + ExchangeData, + V: ExchangeData, + R: ExchangeData + Monoid, + Tr: TraceReader+Clone+'static, + Tr::KeyContainer: BatchContainer, + R: Mul, + FF: Fn(&Tr::Time, &mut Antichain) + 'static, + CF: Fn(Tr::TimeGat<'_>, &Tr::Time) -> bool + 'static, + DOut: Clone+'static, + S: FnMut(&K, &V, Tr::Val<'_>)->DOut+'static, +{ + let output_func = move |session: &mut SessionFor, k: &K, v1: &V, v2: Tr::Val<'_>, initial: &Tr::Time, diff1: &R, output: &mut Vec<(Tr::Time, Tr::Diff)>| { + for (time, diff2) in output.drain(..) { + let diff = diff1.clone() * diff2.clone(); + let dout = (output_func(k, v1, v2), time.clone()); + session.give((dout, initial.clone(), diff)); + } + }; + half_join_internal_unsafe::<_, _, _, _, _, _,_,_, CapacityContainerBuilder>>(stream, arrangement, frontier_func, comparison, |_timer, _count| false, output_func) + .as_collection() +} + +/// A session with lifetime `'a` over timestamp `T` with a container builder `CB`. +/// +/// This is a shorthand primarily for the reson of readability. +type SessionFor<'a, 'b, T, CB> = + Session<'a, 'b, + T, + CB, + Capability, + >; + +/// An unsafe variant of `half_join` where the `output_func` closure takes +/// additional arguments a vector of `time` and `diff` tuples as input and +/// writes its outputs at a container builder. The container builder +/// can, but isn't required to, accept `(data, time, diff)` triplets. +/// This allows for more flexibility, but is more error-prone. +/// +/// This operator responds to inputs of the form +/// +/// ```ignore +/// ((key, val1, time1), initial_time, diff1) +/// ``` +/// +/// where `initial_time` is less or equal to `time1`, and produces as output +/// +/// ```ignore +/// output_func(session, key, val1, val2, initial_time, diff1, &[lub(time1, time2), diff2]) +/// ``` +/// +/// for each `((key, val2), time2, diff2)` present in `arrangement`, where +/// `time2` is less than `initial_time` *UNDER THE TOTAL ORDER ON TIMES*. +/// +/// The `yield_function` allows the caller to indicate when the operator should +/// yield control, as a function of the elapsed time and the number of matched +/// records. Note this is not the number of *output* records, owing mainly to +/// the number of matched records being easiest to record with low overhead. +pub fn half_join_internal_unsafe<'scope, K, V, R, Tr, FF, CF, Y, S, CB>( + stream: VecCollection<'scope, Tr::Time, (K, V, Tr::Time), R>, + mut arrangement: Arranged<'scope, Tr>, + frontier_func: FF, + comparison: CF, + yield_function: Y, + mut output_func: S, +) -> Stream<'scope, Tr::Time, CB::Container> +where + K: Hashable + ExchangeData, + V: ExchangeData, + R: ExchangeData + Monoid, + Tr: TraceReader+Clone+'static, + Tr::KeyContainer: BatchContainer, + FF: Fn(&Tr::Time, &mut Antichain) + 'static, + CF: Fn(Tr::TimeGat<'_>, &Tr::Time) -> bool + 'static, + Y: Fn(std::time::Instant, usize) -> bool + 'static, + S: FnMut(&mut SessionFor, &K, &V, Tr::Val<'_>, &Tr::Time, &R, &mut Vec<(Tr::Time, Tr::Diff)>) + 'static, + CB: ContainerBuilder, +{ + // No need to block physical merging for this operator. + arrangement.trace.set_physical_compaction(Antichain::new().borrow()); + let mut arrangement_trace = Some(arrangement.trace); + let arrangement_stream = arrangement.stream; + + let mut stash = HashMap::new(); + + let exchange = Exchange::new(move |update: &((K, V, Tr::Time),Tr::Time,R)| (update.0).0.hashed().into()); + + // Stash for (time, diff) accumulation. + let mut output_buffer = Vec::new(); + + let scope = stream.scope(); + stream.inner.binary_frontier(arrangement_stream, exchange, Pipeline, "HalfJoin", move |_,info| { + + // Acquire an activator to reschedule the operator when it has unfinished work. + let activator = scope.activator_for(info.address); + + move |(input1, frontier1), (input2, frontier2), output| { + + // drain the first input, stashing requests. + input1.for_each(|capability, data| { + stash.entry(capability.retain(0)) + .or_insert(Vec::new()) + .append(data) + }); + + // Drain input batches; although we do not observe them, we want access to the input + // to observe the frontier and to drive scheduling. + input2.for_each(|_, _| { }); + + // Local variables to track if and when we should exit early. + // The rough logic is that we fully process inputs and set their differences to zero, + // stopping at any point. We clean up all of the zeros in buffers that did any work, + // and reactivate at the end if the yield function still says so. + let mut yielded = false; + let timer = std::time::Instant::now(); + let mut work = 0; + + // New entries to introduce to the stash after processing. + let mut stash_additions = HashMap::new(); + + if let Some(ref mut trace) = arrangement_trace { + + for (capability, proposals) in stash.iter_mut() { + + // Avoid computation if we should already yield. + // TODO: Verify this is correct for TOTAL ORDER. + yielded = yielded || yield_function(timer, work); + if !yielded && !frontier2.less_equal(capability.time()) { + + let frontier = frontier2.frontier(); + + // Update yielded: We can only go from false to {false, true} as + // we're checking that `!yielded` holds before entering this block. + yielded = process_proposals::<_, _, _, _, _, _, _, _>( + &comparison, + &yield_function, + &mut output_func, + &mut output_buffer, + timer, + &mut work, + trace, + proposals, + output.session_with_builder(capability), + frontier + ); + + proposals.retain(|ptd| !ptd.2.is_zero()); + + // Determine the lower bound of remaining update times. + let mut antichain = Antichain::new(); + for (_, initial, _) in proposals.iter() { + antichain.insert(initial.clone()); + } + // Fast path: there is only one element in the antichain. + // All times in `proposals` must be greater or equal to it. + if antichain.len() == 1 && !antichain.less_equal(capability.time()) { + stash_additions + .entry(capability.delayed(&antichain[0])) + .or_insert(Vec::new()) + .append(proposals); + } + else if antichain.len() > 1 { + // Any remaining times should peel off elements from `proposals`. + let mut additions = vec![Vec::new(); antichain.len()]; + for (data, initial, diff) in proposals.drain(..) { + let position = antichain.iter().position(|t| t.less_equal(&initial)).unwrap(); + additions[position].push((data, initial, diff)); + } + for (time, addition) in antichain.into_iter().zip(additions) { + stash_additions + .entry(capability.delayed(&time)) + .or_insert(Vec::new()) + .extend(addition); + } + } + } + } + } + + // If we yielded, re-activate the operator. + if yielded { + activator.activate(); + } + + // drop fully processed capabilities. + stash.retain(|_,proposals| !proposals.is_empty()); + + for (capability, proposals) in stash_additions.into_iter() { + stash.entry(capability).or_insert(Vec::new()).extend(proposals); + } + + // The logical merging frontier depends on both input1 and stash. + let mut frontier = timely::progress::frontier::Antichain::new(); + for time in frontier1.frontier().iter() { + frontier_func(time, &mut frontier); + } + for time in stash.keys() { + frontier_func(time, &mut frontier); + } + arrangement_trace.as_mut().map(|trace| trace.set_logical_compaction(frontier.borrow())); + + if frontier1.is_empty() && stash.is_empty() { + arrangement_trace = None; + } + } + }) +} + +/// Outlined inner loop for `half_join_internal_unsafe` for reasons of performance. +/// +/// Gives Rust/LLVM the opportunity to inline the loop body instead of inlining the loop and +/// leaving all calls in the loop body outlined. +/// +/// Consumes proposals until the yield function returns `true` or all proposals are processed. +/// Leaves a zero diff in place for all proposals that were processed. +/// +/// Returns `true` if the operator should yield. +fn process_proposals( + comparison: &CF, + yield_function: &Y, + output_func: &mut S, + mut output_buffer: &mut Vec<(Tr::Time, Tr::Diff)>, + timer: Instant, + work: &mut usize, + trace: &mut Tr, + proposals: &mut Vec<((K, V, Tr::Time), Tr::Time, R)>, + mut session: SessionFor, + frontier: AntichainRef +) -> bool +where + Tr: TraceReader, + Tr::KeyContainer: BatchContainer, + CF: Fn(Tr::TimeGat<'_>, &Tr::Time) -> bool + 'static, + Y: Fn(Instant, usize) -> bool + 'static, + S: FnMut(&mut SessionFor, &K, &V, Tr::Val<'_>, &Tr::Time, &R, &mut Vec<(Tr::Time, Tr::Diff)>) + 'static, + CB: ContainerBuilder, + K: Ord, + V: Ord, + R: Monoid, +{ + // Sort requests by key for in-order cursor traversal. + consolidate_updates(proposals); + + let (mut cursor, storage) = trace.cursor(); + let mut yielded = false; + + let mut key_con = Tr::KeyContainer::with_capacity(1); + let mut time_con = Tr::TimeContainer::with_capacity(1); + for time in frontier.iter() { + time_con.push_own(time); + } + + // Process proposals one at a time, stopping if we should yield. + for ((ref key, ref val1, ref time), ref initial, ref mut diff1) in proposals.iter_mut() { + // Use TOTAL ORDER to allow the release of `time`. + yielded = yielded || yield_function(timer, *work); + + if !yielded && !(0 .. time_con.len()).any(|i| comparison(time_con.index(i), initial)) { + key_con.clear(); key_con.push_own(&key); + cursor.seek_key(&storage, key_con.index(0)); + if cursor.get_key(&storage) == key_con.get(0) { + while let Some(val2) = cursor.get_val(&storage) { + cursor.map_times(&storage, |t, d| { + if comparison(t, initial) { + let mut t = Tr::owned_time(t); + t.join_assign(time); + output_buffer.push((t, Tr::owned_diff(d))) + } + }); + consolidate(&mut output_buffer); + *work += output_buffer.len(); + output_func(&mut session, key, val1, val2, initial, diff1, &mut output_buffer); + // Defensive clear; we'd expect `output_func` to clear the buffer. + // TODO: Should we assert it is empty? + output_buffer.clear(); + cursor.step_val(&storage); + } + cursor.rewind_vals(&storage); + } + *diff1 = R::zero(); + } + } + + yielded +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join2.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join2.rs new file mode 100644 index 0000000..2c61b8e --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/half_join2.rs @@ -0,0 +1,364 @@ +/// Streaming asymmetric join between updates (K,V1) and an arrangement on (K,V2). +/// +/// The asymmetry is that the join only responds to streamed updates, not to changes in the arrangement. +/// Streamed updates join only with matching arranged updates at lesser times *in the total order*, and +/// subject to a predicate supplied by the user (roughly: strictly less, or not). +/// +/// This behavior can ensure that any pair of matching updates interact exactly once. +/// +/// There are various forms of this operator with tangled closures about how to emit the outputs and +/// wrangle the logical compaction frontier in order to preserve the distinctions around times that are +/// strictly less (conventional compaction logic would collapse unequal times to the frontier, and lose +/// the distiction). +/// +/// The methods also carry an auxiliary time next to the value, which is used to advance the joined times. +/// This is .. a byproduct of wanting to allow advancing times a la `join_function`, without breaking the +/// coupling by total order on "initial time". +/// +/// The doccomments for individual methods are a bit of a mess. Sorry. + +use std::collections::VecDeque; +use std::ops::Mul; + +use timely::ContainerBuilder; +use timely::container::CapacityContainerBuilder; +use timely::dataflow::Stream; +use timely::dataflow::channels::pact::{Pipeline, Exchange}; +use timely::dataflow::operators::Operator; +use timely::PartialOrder; +use timely::progress::{Antichain, ChangeBatch, Timestamp}; +use timely::progress::frontier::MutableAntichain; + +use differential_dataflow::{ExchangeData, VecCollection, AsCollection, Hashable}; +use differential_dataflow::difference::{Monoid, Semigroup}; +use differential_dataflow::lattice::Lattice; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::{Cursor, TraceReader}; +use differential_dataflow::consolidation::{consolidate, consolidate_updates}; +use differential_dataflow::trace::implementations::BatchContainer; + +use timely::dataflow::operators::CapabilitySet; + +/// A binary equijoin that responds to updates on only its first input. +/// +/// This operator responds to inputs of the form +/// +/// ```ignore +/// ((key, val1, time1), initial_time, diff1) +/// ``` +/// +/// where `initial_time` is less or equal to `time1`, and produces as output +/// +/// ```ignore +/// ((output_func(key, val1, val2), lub(time1, time2)), initial_time, diff1 * diff2) +/// ``` +/// +/// for each `((key, val2), time2, diff2)` present in `arrangement`, where +/// `time2` is less than `initial_time` *UNDER THE TOTAL ORDER ON TIMES*. +/// This last constraint is important to ensure that we correctly produce +/// all pairs of output updates across multiple `half_join` operators. +/// +/// Notice that the time is hoisted up into data. The expectation is that +/// once out of the "delta flow region", the updates will be `delay`d to the +/// times specified in the payloads. +pub fn half_join<'scope, K, V, R, Tr, FF, CF, DOut, S>( + stream: VecCollection<'scope, Tr::Time, (K, V, Tr::Time), R>, + arrangement: Arranged<'scope, Tr>, + frontier_func: FF, + comparison: CF, + mut output_func: S, +) -> VecCollection<'scope, Tr::Time, (DOut, Tr::Time), >::Output> +where + K: Hashable + ExchangeData, + V: ExchangeData, + R: ExchangeData + Monoid, + Tr: TraceReader+Clone+'static, + Tr::KeyContainer: BatchContainer, + R: Mul, + FF: Fn(&Tr::Time, &mut Antichain) + 'static, + CF: Fn(Tr::TimeGat<'_>, &Tr::Time) -> bool + 'static, + DOut: Clone+'static, + S: FnMut(&K, &V, Tr::Val<'_>)->DOut+'static, +{ + let output_func = move |builder: &mut CapacityContainerBuilder>, k: &K, v1: &V, v2: Tr::Val<'_>, initial: &Tr::Time, diff1: &R, output: &mut Vec<(Tr::Time, Tr::Diff)>| { + for (time, diff2) in output.drain(..) { + let diff = diff1.clone() * diff2.clone(); + let dout = (output_func(k, v1, v2), time.clone()); + use timely::container::PushInto; + builder.push_into((dout, initial.clone(), diff)); + } + }; + half_join_internal_unsafe::<_, _, _, _, _, _,_,_, CapacityContainerBuilder>>(stream, arrangement, frontier_func, comparison, |_timer, _count| false, output_func) + .as_collection() +} + +/// An unsafe variant of `half_join` where the `output_func` closure takes +/// additional arguments a vector of `time` and `diff` tuples as input and +/// writes its outputs at a container builder. The container builder +/// can, but isn't required to, accept `(data, time, diff)` triplets. +/// This allows for more flexibility, but is more error-prone. +/// +/// This operator responds to inputs of the form +/// +/// ```ignore +/// ((key, val1, time1), initial_time, diff1) +/// ``` +/// +/// where `initial_time` is less or equal to `time1`, and produces as output +/// +/// ```ignore +/// output_func(session, key, val1, val2, initial_time, diff1, &[lub(time1, time2), diff2]) +/// ``` +/// +/// for each `((key, val2), time2, diff2)` present in `arrangement`, where +/// `time2` is less than `initial_time` *UNDER THE TOTAL ORDER ON TIMES*. +/// +/// The `yield_function` allows the caller to indicate when the operator should +/// yield control, as a function of the elapsed time and the number of matched +/// records. Note this is not the number of *output* records, owing mainly to +/// the number of matched records being easiest to record with low overhead. +pub fn half_join_internal_unsafe<'scope, K, V, R, Tr, FF, CF, Y, S, CB>( + stream: VecCollection<'scope, Tr::Time, (K, V, Tr::Time), R>, + mut arrangement: Arranged<'scope, Tr>, + frontier_func: FF, + comparison: CF, + yield_function: Y, + mut output_func: S, +) -> Stream<'scope, Tr::Time, CB::Container> +where + K: Hashable + ExchangeData, + V: ExchangeData, + R: ExchangeData + Monoid, + Tr: TraceReader+Clone+'static, + Tr::KeyContainer: BatchContainer, + FF: Fn(&Tr::Time, &mut Antichain) + 'static, + CF: Fn(Tr::TimeGat<'_>, &Tr::Time) -> bool + 'static, + Y: Fn(std::time::Instant, usize) -> bool + 'static, + S: FnMut(&mut CB, &K, &V, Tr::Val<'_>, &Tr::Time, &R, &mut Vec<(Tr::Time, Tr::Diff)>) + 'static, + CB: ContainerBuilder, +{ + // No need to block physical merging for this operator. + arrangement.trace.set_physical_compaction(Antichain::new().borrow()); + let mut arrangement_trace = Some(arrangement.trace); + let arrangement_stream = arrangement.stream; + + let exchange = Exchange::new(move |update: &((K, V, Tr::Time),Tr::Time,R)| (update.0).0.hashed().into()); + + // Stash for (time, diff) accumulation. + let mut output_buffer = Vec::new(); + + // Unified blobs: each blob holds data in (T, D, R) order, with a stuck_count + // tracking how many elements at the back are not yet eligible for processing. + // The ready prefix is sorted by (D, T, R) for cursor traversal. + let mut blobs: Vec> = Vec::new(); + + let scope = stream.scope(); + stream.inner.binary_frontier(arrangement_stream, exchange, Pipeline, "HalfJoin", move |_,info| { + + // Acquire an activator to reschedule the operator when it has unfinished work. + let activator = scope.activator_for(info.address); + + move |(input1, frontier1), (input2, frontier2), output| { + + // Drain all input into a single buffer. + let mut arriving: Vec<(Tr::Time, (K, V, Tr::Time), R)> = Vec::new(); + let mut caps = CapabilitySet::new(); + input1.for_each(|capability, data| { + caps.insert(capability.retain(0)); + arriving.extend(data.drain(..).map(|(d, t, r)| (t, d, r))); + }); + + // Drain input batches; although we do not observe them, we want access to the input + // to observe the frontier and to drive scheduling. + input2.for_each(|_, _| { }); + + // Local variables to track if and when we should exit early. + let mut yielded = false; + let timer = std::time::Instant::now(); + let mut work = 0; + + if let Some(ref mut trace) = arrangement_trace { + + let frontier = frontier2.frontier(); + + // Determine the total-order minimum of the arrangement frontier, + // used to partition arrivals into immediately-eligible vs stuck. + let mut time_con = Tr::TimeContainer::with_capacity(1); + if let Some(min_time) = frontier.iter().min() { + time_con.push_own(min_time); + } + let eligible = |initial: &Tr::Time| -> bool { + !(0..time_con.len()).any(|i| comparison(time_con.index(i), initial)) + }; + + // Form a new blob from arrivals. + // consolidate_updates sorts by (T, D, R) — the stuck order. + consolidate_updates(&mut arriving); + + if !arriving.is_empty() { + let mut lower = MutableAntichain::new(); + lower.update_iter(arriving.iter().map(|(t, _, _)| (t.clone(), 1))); + let mut blob_caps = CapabilitySet::new(); + for time in lower.frontier().iter() { + blob_caps.insert(caps.delayed(time)); + } + + // Determine how many records are stuck (ineligible). + // Data is sorted by (T, D, R) and eligibility is monotone in T, + // so stuck records form a suffix. + let stuck_count = arriving.iter().rev() + .take_while(|(t, _, _)| !eligible(t)) + .count(); + + let mut data: VecDeque<_> = arriving.into(); + + // Sort the ready prefix by (D, T, R) for cursor traversal. + let ready_len = data.len() - stuck_count; + if ready_len > 0 { + // VecDeque slices: make_contiguous then sort the prefix. + let slice = data.make_contiguous(); + slice[..ready_len].sort_by(|(t1, d1, r1), (t2, d2, r2)| { + (d1, t1, r1).cmp(&(d2, t2, r2)) + }); + } + + blobs.push(Blob { + caps: blob_caps, + lower, + data, + stuck_count, + }); + } + + // Nibble: only when all ready elements have been drained (stuck_count == len), + // check if stuck records have become eligible and promote them. + for blob in blobs.iter_mut().filter(|b| b.stuck_count == b.data.len()) { + + // Count how many stuck records (from the front, which has the + // lowest initial times) are now eligible. + let newly_ready = blob.data.iter().take_while(|(t, _, _)| eligible(t)).count(); + + if newly_ready > 0 { + blob.stuck_count -= newly_ready; + + // Sort the newly-ready prefix by (D, T, R) for cursor traversal. + let slice = blob.data.make_contiguous(); + slice[..newly_ready].sort_by(|(t1, d1, r1), (t2, d2, r2)| { + (d1, t1, r1).cmp(&(d2, t2, r2)) + }); + + // Downgrade capabilities. + let mut new_lower = MutableAntichain::new(); + new_lower.update_iter(blob.data.iter().map(|(t, _, _)| (t.clone(), 1))); + blob.lower = new_lower; + blob.caps.downgrade(&blob.lower.frontier()); + } + } + + // Process ready elements from blobs. + for blob in blobs.iter_mut().filter(|b| b.data.len() > b.stuck_count) { + if yielded { break; } + + let mut builders = (0..blob.caps.len()).map(|_| CB::default()).collect::>(); + + let (mut cursor, storage) = trace.cursor(); + let mut key_con = Tr::KeyContainer::with_capacity(1); + let mut removals: ChangeBatch = ChangeBatch::new(); + + // Process ready elements from the front. + while blob.data.len() > blob.stuck_count { + yielded = yielded || yield_function(timer, work); + if yielded { break; } + + // Peek at the front element. It's in (T, D, R) storage order, + // but the ready prefix has been sorted by (D, T, R). + let (ref initial, (ref key, ref val1, ref time), ref diff1) = blob.data[0]; + + let builder_idx = blob.caps.iter().position(|c| c.time().less_equal(initial)).unwrap(); + + key_con.clear(); key_con.push_own(&key); + cursor.seek_key(&storage, key_con.index(0)); + if cursor.get_key(&storage) == key_con.get(0) { + while let Some(val2) = cursor.get_val(&storage) { + cursor.map_times(&storage, |t, d| { + if comparison(t, initial) { + let mut t = Tr::owned_time(t); + t.join_assign(time); + output_buffer.push((t, Tr::owned_diff(d))) + } + }); + consolidate(&mut output_buffer); + work += output_buffer.len(); + output_func(&mut builders[builder_idx], key, val1, val2, initial, diff1, &mut output_buffer); + output_buffer.clear(); + cursor.step_val(&storage); + } + cursor.rewind_vals(&storage); + } + + while let Some(container) = builders[builder_idx].extract() { + output.session(&blob.caps[builder_idx]).give_container(container); + } + + let (initial, _, _) = blob.data.pop_front().unwrap(); + removals.update(initial, -1); + } + + for builder_idx in 0 .. blob.caps.len() { + while let Some(container) = builders[builder_idx].finish() { + output.session(&blob.caps[builder_idx]).give_container(container); + } + } + + // Apply all removals in bulk and downgrade once. + if blob.data.is_empty() { + // Eagerly release the blob's resources. + blob.lower = MutableAntichain::new(); + blob.caps = CapabilitySet::new(); + blob.data = VecDeque::default(); + } else { + blob.lower.update_iter(removals.drain()); + blob.caps.downgrade(&blob.lower.frontier()); + } + } + + // Remove fully-consumed blobs. + blobs.retain(|blob| !blob.data.is_empty()); + } + + // Re-activate if we have blobs with ready elements to process. + if blobs.iter().any(|b| b.data.len() > b.stuck_count) { + activator.activate(); + } + + // The logical merging frontier depends on input1 and all blobs. + let mut frontier = Antichain::new(); + for time in frontier1.frontier().iter() { + frontier_func(time, &mut frontier); + } + for blob in blobs.iter() { + for cap in blob.caps.iter() { + frontier_func(cap.time(), &mut frontier); + } + } + arrangement_trace.as_mut().map(|trace| trace.set_logical_compaction(frontier.borrow())); + + if frontier1.is_empty() && blobs.is_empty() { + arrangement_trace = None; + } + } + }) +} + +/// A unified blob of updates. Data is stored as `(T, D, R)` tuples in a VecDeque. +/// The last `stuck_count` elements are stuck (not yet eligible for processing), +/// sorted by `(T, D, R)` from consolidation. The ready prefix (everything before +/// the stuck tail) is sorted by `(D, T, R)` for efficient cursor traversal. +/// Ready elements are consumed from the front via `pop_front`. +struct Blob { + caps: CapabilitySet, + lower: MutableAntichain, + data: VecDeque<(T, D, R)>, + /// Number of stuck (ineligible) elements at the back of `data`. + stuck_count: usize, +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/lookup_map.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/lookup_map.rs new file mode 100644 index 0000000..66b1203 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/lookup_map.rs @@ -0,0 +1,142 @@ +use std::collections::HashMap; + +use timely::dataflow::channels::pact::{Pipeline, Exchange}; +use timely::dataflow::operators::Operator; +use timely::PartialOrder; +use timely::progress::Antichain; + +use differential_dataflow::{ExchangeData, VecCollection, AsCollection, Hashable}; +use differential_dataflow::difference::{IsZero, Semigroup, Monoid}; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::{Cursor, TraceReader}; +use differential_dataflow::trace::implementations::BatchContainer; + +/// Proposes extensions to a stream of prefixes. +/// +/// This method takes a stream of prefixes and for each determines a +/// key with `key_selector` and then proposes all pair af the prefix +/// and values associated with the key in `arrangement`. +pub fn lookup_map<'scope, D, K, R, Tr, F, DOut, ROut, S>( + prefixes: VecCollection<'scope, Tr::Time, D, R>, + mut arrangement: Arranged<'scope, Tr>, + key_selector: F, + mut output_func: S, + supplied_key0: K, + supplied_key1: K, + supplied_key2: K, +) -> VecCollection<'scope, Tr::Time, DOut, ROut> +where + Tr: for<'a> TraceReader< + Time: std::hash::Hash, + Diff : Semigroup>+Monoid+ExchangeData, + >+Clone+'static, + Tr::KeyContainer: BatchContainer, + K: Hashable + Ord + 'static, + F: FnMut(&D, &mut K)+Clone+'static, + D: ExchangeData, + R: ExchangeData+Monoid, + DOut: Clone+'static, + ROut: Monoid + 'static, + S: FnMut(&D, &R, Tr::Val<'_>, &Tr::Diff)->(DOut, ROut)+'static, +{ + // No need to block physical merging for this operator. + arrangement.trace.set_physical_compaction(Antichain::new().borrow()); + let mut propose_trace = Some(arrangement.trace); + let propose_stream = arrangement.stream; + + let mut stash = HashMap::new(); + let mut logic1 = key_selector.clone(); + let mut logic2 = key_selector.clone(); + + let mut key: K = supplied_key0; + let exchange = Exchange::new(move |update: &(D,Tr::Time,R)| { + logic1(&update.0, &mut key); + key.hashed().into() + }); + + let mut key1: K = supplied_key1; + let mut key2: K = supplied_key2; + + prefixes.inner.binary_frontier(propose_stream, exchange, Pipeline, "LookupMap", move |_,_| move |(input1, frontier1), (input2, frontier2), output| { + + // drain the first input, stashing requests. + input1.for_each(|capability, data| { + stash.entry(capability.retain(0)) + .or_insert(Vec::new()) + .extend(data.drain(..)) + }); + + // Drain input batches; although we do not observe them, we want access to the input + // to observe the frontier and to drive scheduling. + input2.for_each(|_, _| { }); + + if let Some(ref mut trace) = propose_trace { + + for (capability, prefixes) in stash.iter_mut() { + + // defer requests at incomplete times. + // NOTE: not all updates may be at complete times, but if this test fails then none of them are. + if !frontier2.less_equal(capability.time()) { + + let mut session = output.session(capability); + + // sort requests for in-order cursor traversal. could consolidate? + prefixes.sort_by(|x,y| { + logic2(&x.0, &mut key1); + logic2(&y.0, &mut key2); + key1.cmp(&key2) + }); + + let (mut cursor, storage) = trace.cursor(); + // Key container to stage keys for comparison. + let mut key_con = Tr::KeyContainer::with_capacity(1); + for &mut (ref prefix, ref time, ref mut diff) in prefixes.iter_mut() { + if !frontier2.less_equal(time) { + logic2(prefix, &mut key1); + key_con.clear(); key_con.push_own(&key1); + cursor.seek_key(&storage, key_con.index(0)); + if cursor.get_key(&storage) == Some(key_con.index(0)) { + while let Some(value) = cursor.get_val(&storage) { + let mut count = Tr::Diff::zero(); + cursor.map_times(&storage, |t, d| { + if Tr::owned_time(t).less_equal(time) { count.plus_equals(&d); } + }); + if !count.is_zero() { + let (dout, rout) = output_func(prefix, diff, value, &count); + if !rout.is_zero() { + session.give((dout, time.clone(), rout)); + } + } + cursor.step_val(&storage); + } + cursor.rewind_vals(&storage); + } + *diff = R::zero(); + } + } + + prefixes.retain(|ptd| !ptd.2.is_zero()); + } + } + + } + + // drop fully processed capabilities. + stash.retain(|_,prefixes| !prefixes.is_empty()); + + // The logical merging frontier depends on both input1 and stash. + let mut frontier = timely::progress::frontier::Antichain::new(); + for time in frontier1.frontier().to_vec() { + frontier.insert(time); + } + for key in stash.keys() { + frontier.insert(key.time().clone()); + } + propose_trace.as_mut().map(|trace| trace.set_logical_compaction(frontier.borrow())); + + if frontier1.is_empty() && stash.is_empty() { + propose_trace = None; + } + + }).as_collection() +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/mod.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/mod.rs new file mode 100644 index 0000000..fe15c52 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/mod.rs @@ -0,0 +1,14 @@ +pub mod half_join; +pub mod half_join2; +pub mod lookup_map; + +pub mod count; +pub mod propose; +pub mod validate; + +pub use self::half_join::half_join; +pub use self::half_join2::half_join as half_join2; +pub use self::lookup_map::lookup_map; +pub use self::count::count; +pub use self::propose::{propose, propose_distinct}; +pub use self::validate::validate; diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/propose.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/propose.rs new file mode 100644 index 0000000..187c792 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/propose.rs @@ -0,0 +1,73 @@ +use differential_dataflow::{ExchangeData, VecCollection, Hashable}; +use differential_dataflow::difference::{Semigroup, Monoid, Multiply}; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::TraceReader; + +/// Proposes extensions to a prefix stream. +/// +/// This method takes a collection `prefixes` and an arrangement `arrangement` and for each +/// update in the collection joins with the accumulated arranged records at a time less or +/// equal to that of the update. Note that this is not a join by itself, but can be used to +/// create a join if the `prefixes` collection is also arranged and responds to changes that +/// `arrangement` undergoes. More complicated patterns are also appropriate, as in the case +/// of delta queries. +pub fn propose<'scope, Tr, K, F, P, V>( + prefixes: VecCollection<'scope, Tr::Time, P, Tr::Diff>, + arrangement: Arranged<'scope, Tr>, + key_selector: F, +) -> VecCollection<'scope, Tr::Time, (P, V), Tr::Diff> +where + Tr: for<'a> TraceReader< + ValOwn = V, + Time: std::hash::Hash, + Diff: Monoid+Multiply+ExchangeData+Semigroup>, + >+Clone+'static, + Tr::KeyContainer: differential_dataflow::trace::implementations::BatchContainer, + K: Hashable + Default + Ord + 'static, + F: Fn(&P)->K+Clone+'static, + P: ExchangeData, + V: Clone + 'static, +{ + crate::operators::lookup_map( + prefixes, + arrangement, + move |p: &P, k: &mut K | { *k = key_selector(p); }, + move |prefix, diff, value, sum| ((prefix.clone(), Tr::owned_val(value)), diff.clone().multiply(sum)), + Default::default(), + Default::default(), + Default::default(), + ) +} + +/// Proposes distinct extensions to a prefix stream. +/// +/// Unlike `propose`, this method does not scale the multiplicity of matched +/// prefixes by the number of matches in `arrangement`. This can be useful to +/// avoid the need to prepare an arrangement of distinct extensions. +pub fn propose_distinct<'scope, Tr, K, F, P, V>( + prefixes: VecCollection<'scope, Tr::Time, P, Tr::Diff>, + arrangement: Arranged<'scope, Tr>, + key_selector: F, +) -> VecCollection<'scope, Tr::Time, (P, V), Tr::Diff> +where + Tr: for<'a> TraceReader< + ValOwn = V, + Time: std::hash::Hash, + Diff : Semigroup>+Monoid+Multiply+ExchangeData, + >+Clone+'static, + Tr::KeyContainer: differential_dataflow::trace::implementations::BatchContainer, + K: Hashable + Default + Ord + 'static, + F: Fn(&P)->K+Clone+'static, + P: ExchangeData, + V: Clone + 'static, +{ + crate::operators::lookup_map( + prefixes, + arrangement, + move |p: &P, k: &mut K| { *k = key_selector(p); }, + move |prefix, diff, value, _sum| ((prefix.clone(), Tr::owned_val(value)), diff.clone()), + Default::default(), + Default::default(), + Default::default(), + ) +} diff --git a/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/validate.rs b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/validate.rs new file mode 100644 index 0000000..565bcf5 --- /dev/null +++ b/egglog-experimental/flowlog/vendor/differential-dogs3/src/operators/validate.rs @@ -0,0 +1,38 @@ +use std::hash::Hash; + +use differential_dataflow::{ExchangeData, VecCollection}; +use differential_dataflow::difference::{Semigroup, Monoid, Multiply}; +use differential_dataflow::operators::arrange::Arranged; +use differential_dataflow::trace::TraceReader; + +/// Proposes extensions to a stream of prefixes. +/// +/// This method takes a stream of prefixes and for each determines a +/// key with `key_selector` and then proposes all pair af the prefix +/// and values associated with the key in `arrangement`. +pub fn validate<'scope, K, V, Tr, F, P>( + extensions: VecCollection<'scope, Tr::Time, (P, V), Tr::Diff>, + arrangement: Arranged<'scope, Tr>, + key_selector: F, +) -> VecCollection<'scope, Tr::Time, (P, V), Tr::Diff> +where + Tr: for<'a> TraceReader< + Time: std::hash::Hash, + Diff : Semigroup>+Monoid+Multiply+ExchangeData, + >+Clone+'static, + Tr::KeyContainer: differential_dataflow::trace::implementations::BatchContainer, + K: Ord+Hash+Clone+Default + 'static, + V: ExchangeData+Hash+Default, + F: Fn(&P)->K+Clone+'static, + P: ExchangeData, +{ + crate::operators::lookup_map( + extensions, + arrangement, + move |(pre,val),key| { *key = (key_selector(pre), val.clone()); }, + |(pre,val),r,_,_| ((pre.clone(), val.clone()), r.clone()), + Default::default(), + Default::default(), + Default::default(), + ) +} diff --git a/egglog-experimental/src/keep_best.rs b/egglog-experimental/src/keep_best.rs index 7b701f0..d2b1834 100644 --- a/egglog-experimental/src/keep_best.rs +++ b/egglog-experimental/src/keep_best.rs @@ -9,6 +9,7 @@ use egglog::{ ArcSort, CommandOutput, EGraph, Error, TermDag, TermId, TypeError, UserDefinedCommand, Value, + Write, ast::Expr, extract::{Extractor, TreeAdditiveCostModel}, sort::S, @@ -45,7 +46,7 @@ impl UserDefinedCommand for KeepBestCommand { // Step 4: re-insert the optimal tuples. Evaluate each extracted term // via eval_expr so that constructor sub-terms are re-created bottom-up, - // then stage all target-table inserts in one with_full_state call. + // then stage all target-table inserts in one update call. let mut rows_to_insert: Vec<(String, Vec)> = Vec::new(); for (table_name, extracted_rows, termdag) in &extracted { for term_ids in extracted_rows { @@ -54,11 +55,24 @@ impl UserDefinedCommand for KeepBestCommand { } } - egraph.with_full_state(|mut state| { + egraph.update(|mut state| { for (table_name, values) in &rows_to_insert { - egglog::Write::insert(&mut state, table_name, values.iter().copied()); + let (output, inputs) = values + .split_last() + .expect("keep-best: table row has at least an output column"); + // Function tables set the output at the key; constructor and + // relation tables mint the enode and union it with the output. + match state.set(table_name, egglog::RawValues(inputs.to_vec()), *output) { + Ok(()) => {} + Err(Error::ApiError(egglog::ApiError::WrongSubtype { .. })) => { + let eclass = state.add(table_name, egglog::RawValues(inputs.to_vec()))?; + state.union(eclass, *output)?; + } + Err(e) => return Err(e), + } } - }); + Ok(()) + })?; Ok(vec![]) } @@ -89,8 +103,8 @@ fn collect_and_extract( .collect(); let mut raw_rows: Vec> = Vec::new(); - egraph.function_for_each(table_name, |row| { - raw_rows.push(row.vals.to_vec()); + crate::for_each_full_row(egraph, table_name, |vals| { + raw_rows.push(vals.to_vec()); })?; let extractor = Extractor::compute_costs_from_rootsorts( diff --git a/egglog-experimental/src/lib.rs b/egglog-experimental/src/lib.rs index 79c2907..bf0d2e5 100644 --- a/egglog-experimental/src/lib.rs +++ b/egglog-experimental/src/lib.rs @@ -107,6 +107,30 @@ pub fn new_experimental_egraph() -> EGraph { egraph } +/// Iterate over every row of a table, passing each full row (input columns +/// followed by the output column) to `f`. Works for function, constructor, +/// and relation tables. +pub(crate) fn for_each_full_row( + egraph: &EGraph, + name: &str, + mut f: impl FnMut(&[Value]), +) -> Result<(), Error> { + match egraph.function_entries(name, |e: FunctionEntry<'_>| { + let mut row: Vec = e.inputs.to_vec(); + row.push(e.output); + f(&row); + }) { + Err(Error::ApiError(ApiError::WrongSubtype { .. })) => { + egraph.constructor_enodes(name, |e: Enode<'_>| { + let mut row: Vec = e.children.to_vec(); + row.push(e.eclass); + f(&row); + }) + } + other => other, + } +} + // Create a parser with experimental macros pub fn experimental_parser() -> Parser { let mut parser = Parser::default(); diff --git a/egglog-experimental/src/set_cost.rs b/egglog-experimental/src/set_cost.rs index ce2b4ac..b3f1939 100644 --- a/egglog-experimental/src/set_cost.rs +++ b/egglog-experimental/src/set_cost.rs @@ -1,6 +1,6 @@ use crate::Error; use egglog::{ - CommandOutput, EGraph, TermDag, TermId, UserDefinedCommand, + CommandOutput, EGraph, Read, TermDag, TermId, UserDefinedCommand, ast::*, extract::{CostModel, DefaultCost, Extractor, TreeAdditiveCostModel}, span, @@ -196,21 +196,23 @@ impl CostModel for DynamicCostModel { &self, egraph: &EGraph, func: &egglog::Function, - row: &egglog::FunctionRow<'_>, + enode: &egglog::Enode<'_>, ) -> DefaultCost { let name = get_cost_table_name(func.name()); - let key = row.vals.split_last().unwrap().1; + let key = enode.children; if egraph.get_function(&name).is_some() { egraph - .lookup_function(&name, key) + .read(|rs| rs.lookup(&name, egglog::RawValues(key.to_vec()))) + .ok() + .flatten() .map(|c| { let cost = egraph.value_to_base::(c); assert!(cost >= 0); cost as DefaultCost }) - .unwrap_or_else(|| TreeAdditiveCostModel {}.enode_cost(egraph, func, row)) + .unwrap_or_else(|| TreeAdditiveCostModel {}.enode_cost(egraph, func, enode)) } else { - TreeAdditiveCostModel {}.enode_cost(egraph, func, row) + TreeAdditiveCostModel {}.enode_cost(egraph, func, enode) } } } diff --git a/egglog-experimental/src/sugar/for.rs b/egglog-experimental/src/sugar/for.rs index 23e2521..9235ae4 100644 --- a/egglog-experimental/src/sugar/for.rs +++ b/egglog-experimental/src/sugar/for.rs @@ -42,8 +42,9 @@ impl Macro> for For { body: query, name: rulename, ruleset: ruleset.clone(), - naive: false, + eval_mode: RuleEvalMode::Seminaive, no_decomp: false, + include_subsumed: false, }; Ok(vec![ diff --git a/egglog-experimental/src/table_stats.rs b/egglog-experimental/src/table_stats.rs index b447c0c..7a6848b 100644 --- a/egglog-experimental/src/table_stats.rs +++ b/egglog-experimental/src/table_stats.rs @@ -15,8 +15,7 @@ //! function is reported. use egglog::{ - CommandOutput, EGraph, Error, FunctionRow, TypeError, UserDefinedCommand, Value, ast::Expr, - prelude::Span, + CommandOutput, EGraph, Error, TypeError, UserDefinedCommand, Value, ast::Expr, prelude::Span, }; use egglog_ast::generic_ast::GenericExpr; use std::{ @@ -235,9 +234,8 @@ fn compute_table_stats(egraph: &EGraph, func_name: &str) -> Result>> = HashMap::default(); - egraph.function_for_each(func_name, |row: FunctionRow<'_>| { + crate::for_each_full_row(egraph, func_name, |vals: &[Value]| { size += 1; - let vals = row.vals; debug_assert_eq!(vals.len(), n_cols); for (i, v) in vals.iter().enumerate() { distinct[i].insert(*v); diff --git a/egglog/Cargo.lock b/egglog/Cargo.lock index 20a11e1..f16bdb2 100644 --- a/egglog/Cargo.lock +++ b/egglog/Cargo.lock @@ -468,6 +468,7 @@ dependencies = [ "dyn-clone", "egglog-add-primitive", "egglog-ast", + "egglog-backend-trait", "egglog-bridge", "egglog-core-relations", "egglog-numeric-id", @@ -510,6 +511,17 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "egglog-backend-trait" +version = "2.0.0" +dependencies = [ + "anyhow", + "egglog-bridge", + "egglog-core-relations", + "egglog-numeric-id", + "egglog-reports", +] + [[package]] name = "egglog-bridge" version = "2.0.0" diff --git a/egglog/Cargo.toml b/egglog/Cargo.toml index 04305e0..751444a 100644 --- a/egglog/Cargo.toml +++ b/egglog/Cargo.toml @@ -4,6 +4,7 @@ members = [ "concurrency", "core-relations", "egglog-ast", + "egglog-backend-trait", "egglog-bridge", "numeric-id", "union-find", @@ -78,6 +79,7 @@ egglog-numeric-id = { path = "numeric-id", version = "2.0.0" } egglog-union-find = { path = "union-find", version = "2.0.0" } egglog-add-primitive = { path = "src/sort/add_primitive", version = "2.0.0" } egglog-reports = { path = "egglog-reports", version = "2.0.0" } +egglog-backend-trait = { path = "egglog-backend-trait", version = "2.0.0" } egglog = { path = ".", default-features = false, version = "2.0.0" } ###################### @@ -152,6 +154,7 @@ csv = { workspace = true } egglog-core-relations = { workspace = true } egglog-ast = { workspace = true } egglog-bridge = { workspace = true } +egglog-backend-trait = { workspace = true } egglog-numeric-id = { workspace = true } egglog-add-primitive = { workspace = true } egglog-reports = { workspace = true } diff --git a/egglog/egglog-backend-trait/Cargo.toml b/egglog/egglog-backend-trait/Cargo.toml new file mode 100644 index 0000000..612f6f1 --- /dev/null +++ b/egglog/egglog-backend-trait/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "egglog-backend-trait" +version.workspace = true +edition.workspace = true +description = "A backend-agnostic interface (the `Backend` trait) for driving an egglog e-graph, so third parties can plug in their own execution engine." +repository.workspace = true +keywords.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true } +egglog-bridge = { workspace = true } +egglog-core-relations = { workspace = true } +egglog-numeric-id = { workspace = true } +egglog-reports = { workspace = true } diff --git a/egglog/egglog-backend-trait/src/backend_impl.rs b/egglog/egglog-backend-trait/src/backend_impl.rs new file mode 100644 index 0000000..c7bd443 --- /dev/null +++ b/egglog/egglog-backend-trait/src/backend_impl.rs @@ -0,0 +1,243 @@ +//! `impl Backend for egglog_bridge::EGraph` — the in-memory reference backend. +//! +//! Every method is a thin passthrough to an inherent method on the bridge +//! `EGraph`. Lives in this crate (not in `egglog-bridge`) so the bridge stays +//! free of any dependency on the trait; the orphan rule permits it because the +//! [`Backend`] trait is local here. + +use std::any::Any; +use std::sync::{Arc, RwLock}; + +use anyhow::Result; +use egglog_bridge::{ActionRegistry, EGraph, RuleBuilder}; + +use crate::{ + AtomId, Backend, BaseValues, ColumnTy, ContainerValues, ExecutionState, ExternalFunction, + ExternalFunctionId, FunctionConfig, FunctionId, IterationReport, PanicMsg, QueryEntry, + ReportLevel, RuleBuilderOps, RuleId, ScanEntry, Value, Variable, +}; + +// --------------------------------------------------------------------------- +// RuleBuilderOps for the bridge's RuleBuilder +// --------------------------------------------------------------------------- + +/// Trait-object wrapper around the bridge's [`RuleBuilder`]; every op delegates. +struct BridgeRuleBuilderOps<'a> { + inner: RuleBuilder<'a>, +} + +impl RuleBuilderOps for BridgeRuleBuilderOps<'_> { + fn new_var(&mut self, ty: ColumnTy) -> QueryEntry { + let var: Variable = self.inner.new_var(ty); + QueryEntry::Var(var) + } + + fn new_var_named(&mut self, ty: ColumnTy, name: &str) -> QueryEntry { + self.inner.new_var_named(ty, name) + } + + fn base_values(&self) -> &BaseValues { + self.inner.egraph().base_values() + } + + fn query_table( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + is_subsumed: Option, + ) -> Result<()> { + self.inner + .query_table(func, entries, is_subsumed) + .map(|_| ()) + } + + fn query_prim( + &mut self, + func: ExternalFunctionId, + entries: &[QueryEntry], + ret_ty: ColumnTy, + ) -> Result<()> { + self.inner.query_prim(func, entries, ret_ty) + } + + fn call_external_func( + &mut self, + func: ExternalFunctionId, + args: &[QueryEntry], + ret_ty: ColumnTy, + panic_msg: PanicMsg, + ) -> QueryEntry { + let var = self.inner.call_external_func(func, args, ret_ty, panic_msg); + QueryEntry::Var(var) + } + + fn lookup( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + panic_msg: PanicMsg, + ) -> QueryEntry { + let var = self.inner.lookup(func, entries, panic_msg); + QueryEntry::Var(var) + } + + fn subsume(&mut self, func: FunctionId, entries: &[QueryEntry]) -> Result<()> { + self.inner.subsume(func, entries); + Ok(()) + } + + fn set(&mut self, func: FunctionId, entries: &[QueryEntry]) { + self.inner.set(func, entries); + } + + fn remove(&mut self, func: FunctionId, entries: &[QueryEntry]) { + self.inner.remove(func, entries); + } + + fn union(&mut self, l: QueryEntry, r: QueryEntry) { + self.inner.union(l, r); + } + + fn panic(&mut self, message: String) { + self.inner.panic(message); + } + + fn new_panic(&mut self, message: String) -> ExternalFunctionId { + self.inner.new_panic(message) + } + + fn set_no_decomp(&mut self, no_decomp: bool) { + self.inner.set_no_decomp(no_decomp); + } + + fn build(self: Box) -> Result { + Ok(self.inner.build()) + } + + fn query_table_branch( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + is_subsumed: Option, + ) -> Result { + self.inner.query_table(func, entries, is_subsumed) + } + + fn set_union_branches(&mut self, branch_atoms: Vec>, output_vars: Vec) { + self.inner.set_union_branches(branch_atoms, output_vars); + } + + fn backend_any(&self) -> Option<&dyn Any> { + Some(self.inner.egraph()) + } +} + +// --------------------------------------------------------------------------- +// Backend for the bridge EGraph +// --------------------------------------------------------------------------- + +impl Backend for EGraph { + fn add_table(&mut self, config: FunctionConfig) -> FunctionId { + EGraph::add_table(self, config) + } + + fn table_size(&self, table: FunctionId) -> usize { + EGraph::table_size(self, table) + } + + fn clear_table(&mut self, func: FunctionId) { + EGraph::clear_table(self, func); + } + + fn for_each_dyn(&self, table: FunctionId, f: &mut dyn for<'r> FnMut(ScanEntry<'r>)) { + EGraph::for_each(self, table, f); + } + + fn for_each_while_dyn( + &self, + table: FunctionId, + f: &mut dyn for<'r> FnMut(ScanEntry<'r>) -> bool, + ) { + EGraph::for_each_while(self, table, f); + } + + fn get_canon_repr(&self, val: Value, ty: ColumnTy) -> Value { + EGraph::get_canon_repr(self, val, ty) + } + + fn base_values(&self) -> &BaseValues { + EGraph::base_values(self) + } + + fn base_values_mut(&mut self) -> &mut BaseValues { + EGraph::base_values_mut(self) + } + + fn container_values(&self) -> &ContainerValues { + EGraph::container_values(self) + } + + fn with_execution_state_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)) { + EGraph::with_execution_state(self, |es| f(es)); + } + + fn with_execution_state_tracked_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)) -> bool { + EGraph::with_execution_state_tracked(self, |es| f(es)).1 + } + + fn new_rule<'a>(&'a mut self, desc: &str, seminaive: bool) -> Box { + let inner = EGraph::new_rule(self, desc, seminaive); + Box::new(BridgeRuleBuilderOps { inner }) + } + + fn free_rule(&mut self, id: RuleId) { + EGraph::free_rule(self, id); + } + + fn run_rules(&mut self, rules: &[RuleId]) -> Result { + EGraph::run_rules(self, rules) + } + + fn flush_updates(&mut self) -> bool { + EGraph::flush_updates(self) + } + + fn register_external_func( + &mut self, + func: Box, + ) -> ExternalFunctionId { + EGraph::register_external_func(self, func) + } + + fn free_external_func(&mut self, func: ExternalFunctionId) { + EGraph::free_external_func(self, func); + } + + fn new_panic(&mut self, message: String) -> ExternalFunctionId { + EGraph::new_panic(self, message) + } + + fn set_report_level(&mut self, level: ReportLevel) { + EGraph::set_report_level(self, level); + } + + fn dump_debug_info(&self) { + EGraph::dump_debug_info(self); + } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + fn action_registry(&self) -> &Arc> { + EGraph::action_registry(self) + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } +} diff --git a/egglog/egglog-backend-trait/src/lib.rs b/egglog/egglog-backend-trait/src/lib.rs new file mode 100644 index 0000000..7668d52 --- /dev/null +++ b/egglog/egglog-backend-trait/src/lib.rs @@ -0,0 +1,451 @@ +//! # egglog-backend-trait +//! +//! A small, backend-agnostic interface for driving an egglog e-graph. The +//! frontend `egglog::EGraph` holds a `Box` and performs all state +//! access through the [`Backend`] trait, so a third party can plug in their own +//! execution engine (a relational database, a dataflow system, …) without +//! forking the frontend. +//! +//! ## What a backend must provide +//! +//! [`Backend`] is deliberately small. The *required* surface is the handful of +//! operations the frontend actually needs: register functions ([`Backend::add_table`]), +//! build and run rules ([`Backend::new_rule`] / [`Backend::run_rules`]), read +//! tables back ([`Backend::for_each_while_dyn`] / [`Backend::table_size`]), +//! canonicalize ids ([`Backend::get_canon_repr`]), reach the value registries +//! ([`Backend::base_values`] / [`Backend::container_values`]), and register +//! primitives ([`Backend::register_external_func`]). +//! +//! Rules are built through [`RuleBuilderOps`], which mirrors the reference +//! `egglog_bridge::RuleBuilder` one-for-one: a backend accumulates the calls +//! into its own IR and finalizes on [`RuleBuilderOps::build`]. +//! +//! ## Advanced features are optional +//! +//! Capabilities that not every backend can offer — the seminaive-safe +//! [`ActionRegistry`] ([`Backend::action_registry`]), container sorts, +//! subsumption — are exposed as methods with **default** bodies (return `false` +//! / `unimplemented!`). An implementer overrides only what it supports and +//! advertises it through the `supports_*` flags; the frontend gates on those. +//! +//! ## Ergonomic sugar +//! +//! The object-safe trait methods take erased closures (`&mut dyn FnMut(..)`) and +//! carry a `_dyn` suffix. Generic, ergonomic wrappers ([`Backend::for_each`], +//! [`Backend::with_execution_state`], [`Backend::base_value_constant`], …) are +//! provided as inherent methods on `dyn Backend` so call sites read naturally. +//! +//! ## Relationship to `egglog-bridge` +//! +//! This crate depends on `egglog-bridge` and re-exports its vocabulary types +//! ([`FunctionConfig`], [`MergeFn`], [`ColumnTy`], [`Value`], …) so an +//! implementer imports them from here. It also provides the reference +//! `impl Backend for egglog_bridge::EGraph` (see `backend_impl`), leaving the +//! bridge crate itself untouched. + +use std::any::Any; +use std::sync::{Arc, RwLock}; + +use anyhow::Result; + +mod backend_impl; + +// --------------------------------------------------------------------------- +// Re-exports: the shared vocabulary a backend implementer works with. +// --------------------------------------------------------------------------- + +pub use egglog_bridge::{ + ActionRegistry, AtomId, ColumnTy, DefaultVal, FunctionConfig, FunctionId, MergeFn, QueryEntry, + RuleId, ScanEntry, Variable, +}; +pub use egglog_core_relations::{ + BaseValue, BaseValueId, BaseValues, ContainerValue, ContainerValues, ExecutionState, + ExternalFunction, ExternalFunctionId, Value, +}; +pub use egglog_reports::{IterationReport, ReportLevel}; + +/// A lazily-computed failure message for RHS lookups / external calls. +/// +/// Only built if the lookup or call actually fails at runtime; deferring it +/// avoids formatting a `Span` (an `O(file)` scan) for every action in every +/// generated rule. Boxed so the trait stays object-safe. +pub type PanicMsg = Box String + Send>; + +// --------------------------------------------------------------------------- +// The `Backend` trait +// --------------------------------------------------------------------------- + +/// A backend that stores an egglog e-graph and runs rules over it. +/// +/// The frontend holds a `Box` and drives everything through this +/// trait. Implementations: the in-memory reference `egglog_bridge::EGraph` +/// (this crate's `backend_impl`), and out-of-tree engines such as the FlowLog +/// (differential-dataflow) backend in egglog-experimental. +/// +/// See the crate docs for which methods are required vs. optional and how the +/// ergonomic sugar on `dyn Backend` relates to the `_dyn` methods here. +pub trait Backend: Send + Sync { + // -- table lifecycle ---------------------------------------------------- + + /// Register a function / relation / constructor and return its handle. + fn add_table(&mut self, config: FunctionConfig) -> FunctionId; + + /// Number of rows currently in the given function's table. + fn table_size(&self, table: FunctionId) -> usize; + + /// Remove every row from the given function's table. + fn clear_table(&mut self, func: FunctionId); + + // -- iteration (object-safe; see `for_each` / `for_each_while` sugar) ---- + + /// Call `f` on every row in `table`. Object-safe form; prefer the + /// [`Backend::for_each`] sugar on `dyn Backend`. + fn for_each_dyn(&self, table: FunctionId, f: &mut dyn for<'r> FnMut(ScanEntry<'r>)); + + /// Iterate rows in `table`, stopping early when `f` returns `false`. + /// Object-safe form; prefer the [`Backend::for_each_while`] sugar. + fn for_each_while_dyn( + &self, + table: FunctionId, + f: &mut dyn for<'r> FnMut(ScanEntry<'r>) -> bool, + ); + + // -- direct access ------------------------------------------------------ + + /// Canonical representative of `val` under `ty` (union-find find for + /// [`ColumnTy::Id`]; identity for a base value). + fn get_canon_repr(&self, val: Value, ty: ColumnTy) -> Value; + + /// The backend's base-value (primitive) registry. + fn base_values(&self) -> &BaseValues; + + /// Mutable base-value registry (used to register new base types). + fn base_values_mut(&mut self) -> &mut BaseValues; + + /// The backend's container-value registry. + fn container_values(&self) -> &ContainerValues; + + // -- execution state (object-safe; see `with_execution_state` sugar) ----- + + /// Run `f` against a fresh execution state (for staging updates / calling + /// primitives). Object-safe form; prefer [`Backend::with_execution_state`]. + fn with_execution_state_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)); + + /// Like [`Backend::with_execution_state_dyn`], but returns whether `f` + /// staged any mutation. Object-safe form; prefer + /// [`Backend::with_execution_state_tracked`]. + fn with_execution_state_tracked_dyn(&self, f: &mut dyn FnMut(&mut ExecutionState<'_>)) -> bool; + + // -- rule management ---------------------------------------------------- + + /// Begin building a rule. Populate via [`RuleBuilderOps`] and finalize with + /// [`RuleBuilderOps::build`]. + fn new_rule<'a>(&'a mut self, desc: &str, seminaive: bool) -> Box; + + /// Drop a registered rule. + fn free_rule(&mut self, id: RuleId); + + /// Run one iteration of the given rule set. + fn run_rules(&mut self, rules: &[RuleId]) -> Result; + + /// Drain staged inserts and rebuild if the union-find changed. Returns + /// whether the database changed. + fn flush_updates(&mut self) -> bool; + + // -- primitives --------------------------------------------------------- + + /// Register a user-defined primitive (`ExternalFunction`). + fn register_external_func( + &mut self, + func: Box, + ) -> ExternalFunctionId; + + /// Drop a user-defined primitive. + fn free_external_func(&mut self, func: ExternalFunctionId); + + /// Register a deferred-panic primitive; returns its id. + fn new_panic(&mut self, message: String) -> ExternalFunctionId; + + // -- diagnostics -------------------------------------------------------- + + /// Set the verbosity of the per-iteration timing report. + fn set_report_level(&mut self, level: ReportLevel); + + /// Dump the database state to the log channel (debug only). + fn dump_debug_info(&self); + + // -- cloning ------------------------------------------------------------ + + /// Deep-clone the backend (used for push/pop snapshots). + fn clone_boxed(&self) -> Box; + + // -- optional / advanced (default-provided) ----------------------------- + + /// Handle to the seminaive-safe [`ActionRegistry`] that registry-backed + /// primitives dispatch through. + /// + /// Only backends whose primitives run against an in-memory + /// [`ExecutionState`] provide this (the reference bridge). Backends that + /// have no such registry return [`Backend::supports_action_registry`] = + /// `false` and leave this default (the frontend routes their primitives + /// through a registry-free path). + fn action_registry(&self) -> &Arc> { + unimplemented!("this backend has no action registry") + } + + /// Whether this backend supports the `subsume` action / `is_subsumed` + /// filter. + fn supports_subsumption(&self) -> bool { + true + } + + /// Whether this backend supports `Vec` / `Set` / `Map` / `MultiSet` + /// container sorts. + fn supports_containers(&self) -> bool { + true + } + + /// Whether this backend exposes an in-memory [`ActionRegistry`] + /// ([`Backend::action_registry`]). + fn supports_action_registry(&self) -> bool { + true + } + + // -- concrete-backend access (escape hatch) ----------------------------- + + /// `&self` as `&dyn Any`, to downcast to the concrete backend type. Used + /// by the container-registration sugar to reach the reference bridge. + /// Implementations return `self`. + fn as_any(&self) -> &dyn Any; + + /// Mutable counterpart of [`Backend::as_any`]. Implementations return + /// `self`. + fn as_any_mut(&mut self) -> &mut dyn Any; +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_boxed() + } +} + +// --------------------------------------------------------------------------- +// Ergonomic sugar: `BackendExt` +// --------------------------------------------------------------------------- + +/// Ergonomic, generic wrappers over the object-safe [`Backend`] surface. +/// +/// Provided by a blanket impl for every `B: Backend + ?Sized`, so the sugar is +/// available on `dyn Backend`, `Box`, and any concrete backend. +/// Import this trait to call `for_each` / `with_execution_state` / +/// `base_value_constant` / `register_container_ty` on a backend. +pub trait BackendExt { + /// Call `f` on every row in `table`. + fn for_each(&self, table: FunctionId, f: impl for<'r> FnMut(ScanEntry<'r>)); + + /// Iterate rows in `table`, stopping when `f` returns `false`. + fn for_each_while(&self, table: FunctionId, f: impl for<'r> FnMut(ScanEntry<'r>) -> bool); + + /// Run `f` against a fresh execution state and return its result. + fn with_execution_state(&self, f: impl FnOnce(&mut ExecutionState<'_>) -> R) -> R; + + /// Run `f` against a fresh execution state, also reporting whether `f` + /// staged any mutation. + fn with_execution_state_tracked( + &self, + f: impl FnOnce(&mut ExecutionState<'_>) -> R, + ) -> (R, bool); + + /// Build a [`QueryEntry`] constant for a typed base value. + fn base_value_constant(&self, x: T) -> QueryEntry; + + /// Register a container-value type `C`. + /// + /// Container registration wires the container into the backend's rebuild + /// loop, which requires backend-internal state. Only the reference bridge + /// implements it today; on a backend that does not support containers this + /// is a no-op (such programs never construct containers). + fn register_container_ty(&mut self); + + /// Intern a container value `C`, returning its `Value` handle. + fn get_container_value(&mut self, val: C) -> Value; +} + +impl BackendExt for B { + fn for_each(&self, table: FunctionId, mut f: impl for<'r> FnMut(ScanEntry<'r>)) { + self.for_each_dyn(table, &mut f); + } + + fn for_each_while(&self, table: FunctionId, mut f: impl for<'r> FnMut(ScanEntry<'r>) -> bool) { + self.for_each_while_dyn(table, &mut f); + } + + fn with_execution_state(&self, f: impl FnOnce(&mut ExecutionState<'_>) -> R) -> R { + let mut f = Some(f); + let mut out: Option = None; + self.with_execution_state_dyn(&mut |es| { + let f = f.take().expect("with_execution_state closure called once"); + out = Some(f(es)); + }); + out.expect("with_execution_state_dyn must invoke its closure exactly once") + } + + fn with_execution_state_tracked( + &self, + f: impl FnOnce(&mut ExecutionState<'_>) -> R, + ) -> (R, bool) { + let mut f = Some(f); + let mut out: Option = None; + let mutated = self.with_execution_state_tracked_dyn(&mut |es| { + let f = f + .take() + .expect("with_execution_state_tracked closure called once"); + out = Some(f(es)); + }); + ( + out.expect("with_execution_state_tracked_dyn must invoke its closure exactly once"), + mutated, + ) + } + + fn base_value_constant(&self, x: T) -> QueryEntry { + QueryEntry::Const { + val: self.base_values().get(x), + ty: ColumnTy::Base(self.base_values().get_ty::()), + } + } + + fn register_container_ty(&mut self) { + if let Some(bridge) = self.as_any_mut().downcast_mut::() { + bridge.register_container_ty::(); + } else { + assert!( + !self.supports_containers(), + "backend advertises container support but does not route register_container_ty" + ); + } + } + + fn get_container_value(&mut self, val: C) -> Value { + if let Some(bridge) = self.as_any_mut().downcast_mut::() { + bridge.get_container_value::(val) + } else { + panic!("get_container_value is only supported on the reference bridge backend") + } + } +} + +// --------------------------------------------------------------------------- +// `RuleBuilderOps` — mirrors `egglog_bridge::RuleBuilder` +// --------------------------------------------------------------------------- + +/// Operations on an in-progress rule. Mirrors the public surface of +/// `egglog_bridge::RuleBuilder`: the reference impl is a thin passthrough; an +/// out-of-tree backend accumulates these calls into its own IR and finalizes on +/// [`RuleBuilderOps::build`]. +pub trait RuleBuilderOps { + /// Bind a new variable of the given type. + fn new_var(&mut self, ty: ColumnTy) -> QueryEntry; + + /// Bind a new variable of the given type with a display name. + fn new_var_named(&mut self, ty: ColumnTy, name: &str) -> QueryEntry; + + /// The base-value registry, for building typed constants during rule + /// construction. (Replaces reaching through the concrete backend.) + fn base_values(&self) -> &BaseValues; + + /// Add a table body atom. The final entry is the function's return value; + /// `is_subsumed`, when `Some`, filters on the subsumption bit. + fn query_table( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + is_subsumed: Option, + ) -> Result<()>; + + /// Add a primitive body atom. The final entry is the return value. + fn query_prim( + &mut self, + func: ExternalFunctionId, + entries: &[QueryEntry], + ret_ty: ColumnTy, + ) -> Result<()>; + + /// Call an external function in the RHS, panicking with `panic_msg` (built + /// lazily) on failure. Returns the result variable. + fn call_external_func( + &mut self, + func: ExternalFunctionId, + args: &[QueryEntry], + ret_ty: ColumnTy, + panic_msg: PanicMsg, + ) -> QueryEntry; + + /// RHS: look up `func(entries)` with the function's configured default on + /// miss. + fn lookup( + &mut self, + func: FunctionId, + entries: &[QueryEntry], + panic_msg: PanicMsg, + ) -> QueryEntry; + + /// RHS: subsume the row keyed by `entries` in `func`. + fn subsume(&mut self, func: FunctionId, entries: &[QueryEntry]) -> Result<()>; + + /// RHS: set `func(entries[..n-1])` to `entries[n-1]`. + fn set(&mut self, func: FunctionId, entries: &[QueryEntry]); + + /// RHS: remove the row keyed by `entries` from `func`. + fn remove(&mut self, func: FunctionId, entries: &[QueryEntry]); + + /// RHS: union two values in the union-find. + fn union(&mut self, l: QueryEntry, r: QueryEntry); + + /// RHS: panic with the given message. + fn panic(&mut self, message: String); + + /// Register a deferred-panic external function on the backend and return + /// its id (used, e.g., to bake a panic id into a wrapped function). + fn new_panic(&mut self, message: String) -> ExternalFunctionId; + + /// Skip tree-decomposition during query planning for this rule + /// (`:no-decomp`). Default no-op for backends that don't decompose. + fn set_no_decomp(&mut self, _no_decomp: bool) {} + + /// Finalize the rule, returning its [`RuleId`]. + fn build(self: Box) -> Result; + + // -- optional: disjunctive (`or`) patterns ------------------------------ + + /// Like [`RuleBuilderOps::query_table`], but returns the atom's [`AtomId`] + /// so it can be grouped into an `or` union branch via + /// [`RuleBuilderOps::set_union_branches`]. Only backends that support + /// disjunctive query patterns implement this. + fn query_table_branch( + &mut self, + _func: FunctionId, + _entries: &[QueryEntry], + _is_subsumed: Option, + ) -> Result { + Err(anyhow::anyhow!( + "disjunctive (`or`) query patterns are not supported on this backend" + )) + } + + /// Fuse the given branches (each a set of body atoms identified by + /// [`AtomId`]) into a disjunctive union over the shared `output_vars`. + /// Only backends that support `or` patterns implement this. + fn set_union_branches(&mut self, _branch_atoms: Vec>, _output_vars: Vec) { + unimplemented!("disjunctive (`or`) query patterns are not supported on this backend") + } + + // -- optional: bridge-only escape hatch --------------------------------- + + /// The underlying backend as `&dyn Any` during rule building, for + /// bridge-only features such as `unstable-fn`'s `TableAction` capture. + /// Returns `None` on backends that do not expose it. + fn backend_any(&self) -> Option<&dyn std::any::Any> { + None + } +} diff --git a/egglog/src/lib.rs b/egglog/src/lib.rs index 8b5f802..d7bc75a 100644 --- a/egglog/src/lib.rs +++ b/egglog/src/lib.rs @@ -39,6 +39,10 @@ pub use egglog_add_primitive::add_primitive_with_validator; use egglog_ast::generic_ast::{Change, GenericExpr, Literal}; use egglog_ast::span::Span; use egglog_ast::util::ListDisplay; +use egglog_backend_trait::RuleBuilderOps; +/// The pluggable backend interface. Re-exported so downstream crates can +/// implement their own backend (see [`EGraph::with_backend`]). +pub use egglog_backend_trait::{Backend, BackendExt}; use egglog_bridge::{ColumnTy, QueryEntry}; use egglog_core_relations as core_relations; use egglog_numeric_id as numeric_id; @@ -284,7 +288,7 @@ dyn_clone::clone_trait_object!(ExtensionStateValue); #[derive(Clone)] pub struct EGraph { - backend: egglog_bridge::EGraph, + backend: Box, pub parser: Parser, names: check_shadowing::Names, /// pushed_egraph forms a linked list of pushed egraphs. @@ -395,10 +399,22 @@ impl Debug for Function { impl Default for EGraph { fn default() -> Self { + Self::with_backend(Box::new(egglog_bridge::EGraph::default())) + } +} + +impl EGraph { + /// Construct an `EGraph` backed by the given [`Backend`] implementation. + /// + /// [`EGraph::default`] uses the in-memory reference backend + /// (`egglog_bridge::EGraph`); downstream crates can supply their own + /// backend (e.g. a differential-dataflow engine) by implementing + /// [`Backend`] and passing it here. + pub fn with_backend(backend: Box) -> Self { let mut parser = Parser::default(); let proof_state = EncodingState::new(&mut parser.symbol_gen); let mut eg = Self { - backend: Default::default(), + backend, parser, names: Default::default(), pushed_egraph: Default::default(), @@ -746,7 +762,7 @@ impl EGraph { ) -> Result { match expr { GenericExpr::Lit(_, literal) => { - let val = literal_to_value(&self.backend, literal); + let val = literal_to_value(self.backend.base_values(), literal); Ok(egglog_bridge::MergeFn::Const(val)) } GenericExpr::Var(span, resolved_var) => match resolved_var.name.as_str() { @@ -776,17 +792,24 @@ impl EGraph { "expected string literal after `unstable-fn`".into(), )); }; + let panic_id = self + .backend + .action_registry() + .read() + .unwrap() + .default_panic_id(); + let bridge = self + .backend + .as_any() + .downcast_ref::() + .expect("`unstable-fn` is only supported on the reference bridge backend"); let resolved = resolve_function_container_target_with_context( - &self.backend, + bridge, &self.functions, &self.type_info, name, p, - self.backend - .action_registry() - .read() - .unwrap() - .default_panic_id(), + panic_id, )?; translated_args[0] = egglog_bridge::MergeFn::Const(self.backend.base_values().get(resolved)); @@ -827,7 +850,7 @@ impl EGraph { schema: input .iter() .chain([&output]) - .map(|sort| sort.column_ty(&self.backend)) + .map(|sort| sort.column_ty(self.backend.base_values())) .collect(), default: match decl.subtype { FunctionSubtype::Constructor => DefaultVal::FreshId, @@ -1614,8 +1637,13 @@ impl EGraph { "unstable-fn over `{name}` was applied in a context where its wrapped \ function is not valid for this call site, if in a rule, add :naive." )); + let bridge = self + .backend + .as_any() + .downcast_ref::() + .expect("`unstable-fn` is only supported on the reference bridge backend"); let resolved_function = resolve_function_container_target_with_context( - &self.backend, + bridge, &self.functions, &self.type_info, name, @@ -1705,7 +1733,7 @@ impl EGraph { ext_id, &[arg], egglog_bridge::ColumnTy::Base(unit_id), - || "this function will never panic".to_string(), + Box::new(|| "this function will never panic".to_string()), ); let id = translator.build(); @@ -1770,11 +1798,12 @@ impl EGraph { true, // global query: Read context (may read the DB) ); translator.query(&query, true); - translator - .rb - .call_external_func(ext_id, &[], egglog_bridge::ColumnTy::Id, || { - "this function will never panic".to_string() - }); + translator.rb.call_external_func( + ext_id, + &[], + egglog_bridge::ColumnTy::Id, + Box::new(|| "this function will never panic".to_string()), + ); let id = translator.build(); let run_result = self.backend.run_rules(&[id]); self.backend.free_rule(id); @@ -2124,7 +2153,12 @@ impl EGraph { let num_facts = parsed_contents.len(); - let table_action = egglog_bridge::TableAction::new(&self.backend, func.backend_id); + let bridge = self + .backend + .as_any() + .downcast_ref::() + .expect("loading facts from a file requires the reference bridge backend"); + let table_action = egglog_bridge::TableAction::new(bridge, func.backend_id); if function_type.subtype != FunctionSubtype::Constructor { self.backend.with_execution_state(|es| { @@ -2753,7 +2787,7 @@ fn resolve_function_container_target_with_context( } struct BackendRule<'a> { - rb: egglog_bridge::RuleBuilder<'a>, + rb: Box, entries: HashMap, functions: &'a IndexMap, type_info: &'a TypeInfo, @@ -2766,7 +2800,7 @@ struct BackendRule<'a> { impl<'a> BackendRule<'a> { fn new( - rb: egglog_bridge::RuleBuilder<'a>, + rb: Box, functions: &'a IndexMap, type_info: &'a TypeInfo, requires_read_context: bool, @@ -2811,10 +2845,11 @@ impl<'a> BackendRule<'a> { self.entries .entry(x.clone()) .or_insert_with(|| match x { - core::GenericAtomTerm::Var(_, v) => self - .rb - .new_var_named(v.sort.column_ty(self.rb.egraph()), &v.name), - core::GenericAtomTerm::Literal(_, l) => literal_to_entry(self.rb.egraph(), l), + core::GenericAtomTerm::Var(_, v) => { + let ty = v.sort.column_ty(self.rb.base_values()); + self.rb.new_var_named(ty, &v.name) + } + core::GenericAtomTerm::Literal(_, l) => literal_to_entry(self.rb.base_values(), l), core::GenericAtomTerm::Global(..) => { panic!("Globals should have been desugared") } @@ -2852,8 +2887,13 @@ impl<'a> BackendRule<'a> { "unstable-fn over `{name}` was applied in a context where its wrapped \ function is not valid for this call site, if in a rule, add :naive." )); + let bridge = self + .rb + .backend_any() + .and_then(|a| a.downcast_ref::()) + .expect("`unstable-fn` is only supported on the reference bridge backend"); let resolved = resolve_function_container_target_with_context( - self.rb.egraph(), + bridge, self.functions, self.type_info, name, @@ -2862,14 +2902,11 @@ impl<'a> BackendRule<'a> { ) .unwrap_or_else(|err| panic!("{err}")); - qe_args[0] = self.rb.egraph().base_value_constant(resolved); + qe_args[0] = base_constant(self.rb.base_values(), resolved); } - ( - resolved_id, - qe_args, - prim.output().column_ty(self.rb.egraph()), - ) + let output_ty = prim.output().column_ty(self.rb.base_values()); + (resolved_id, qe_args, output_ty) } fn args<'b>( @@ -2919,7 +2956,7 @@ impl<'a> BackendRule<'a> { true => None, false => Some(false), }; - atom_ids.push(self.rb.query_table(f, &args, is_subsumed).unwrap()); + atom_ids.push(self.rb.query_table_branch(f, &args, is_subsumed).unwrap()); } ResolvedCall::Primitive(p) => { return Err(Error::BackendError(format!( @@ -2961,21 +2998,30 @@ impl<'a> BackendRule<'a> { let f = self.func(f); let args = self.args(args); let span = span.clone(); - self.rb.lookup(f, &args, move || { - format!("{span}: lookup of function {name} failed") - }) + self.rb.lookup( + f, + &args, + Box::new(move || { + format!("{span}: lookup of function {name} failed") + }), + ) } ResolvedCall::Primitive(p) => { let name = p.name().to_owned(); let ctx = self.action_context(); let (p, args, ty) = self.prim(p, args, ctx); let span = span.clone(); - self.rb.call_external_func(p, &args, ty, move || { - format!("{span}: call of primitive {name} failed") - }) + self.rb.call_external_func( + p, + &args, + ty, + Box::new(move || { + format!("{span}: call of primitive {name} failed") + }), + ) } }; - self.entries.insert(v, y.into()); + self.entries.insert(v, y); } core::GenericCoreAction::LetAtomTerm(span, v, x) => { let v = core::GenericAtomTerm::Var(span.clone(), v.clone()); @@ -2999,7 +3045,7 @@ impl<'a> BackendRule<'a> { let args = self.args(args); match change { Change::Delete => self.rb.remove(f, &args), - Change::Subsume if can_subsume => self.rb.subsume(f, &args), + Change::Subsume if can_subsume => self.rb.subsume(f, &args).unwrap(), Change::Subsume => { return Err(Error::SubsumeMergeError(name, span.clone())); } @@ -3018,7 +3064,9 @@ impl<'a> BackendRule<'a> { } fn build(self) -> egglog_bridge::RuleId { - self.rb.build() + self.rb + .build() + .unwrap_or_else(|err| panic!("rule build failed: {err}")) } } @@ -3085,23 +3133,32 @@ fn common_branch_vars(branches: &[Vec]) -> Vec { common.into_values().collect() } -fn literal_to_entry(egraph: &egglog_bridge::EGraph, l: &Literal) -> QueryEntry { +/// Build a [`QueryEntry`] constant for a typed base value, given the backend's +/// [`BaseValues`] registry. +fn base_constant(base_values: &BaseValues, x: T) -> QueryEntry { + QueryEntry::Const { + val: base_values.get(x), + ty: ColumnTy::Base(base_values.get_ty::()), + } +} + +fn literal_to_entry(base_values: &BaseValues, l: &Literal) -> QueryEntry { match l { - Literal::Int(x) => egraph.base_value_constant::(*x), - Literal::Float(x) => egraph.base_value_constant::(x.into()), - Literal::String(x) => egraph.base_value_constant::(sort::S::new(x.clone())), - Literal::Bool(x) => egraph.base_value_constant::(*x), - Literal::Unit => egraph.base_value_constant::<()>(()), + Literal::Int(x) => base_constant::(base_values, *x), + Literal::Float(x) => base_constant::(base_values, x.into()), + Literal::String(x) => base_constant::(base_values, sort::S::new(x.clone())), + Literal::Bool(x) => base_constant::(base_values, *x), + Literal::Unit => base_constant::<()>(base_values, ()), } } -fn literal_to_value(egraph: &egglog_bridge::EGraph, l: &Literal) -> Value { +fn literal_to_value(base_values: &BaseValues, l: &Literal) -> Value { match l { - Literal::Int(x) => egraph.base_values().get::(*x), - Literal::Float(x) => egraph.base_values().get::(x.into()), - Literal::String(x) => egraph.base_values().get::(sort::S::new(x.clone())), - Literal::Bool(x) => egraph.base_values().get::(*x), - Literal::Unit => egraph.base_values().get::<()>(()), + Literal::Int(x) => base_values.get::(*x), + Literal::Float(x) => base_values.get::(x.into()), + Literal::String(x) => base_values.get::(sort::S::new(x.clone())), + Literal::Bool(x) => base_values.get::(*x), + Literal::Unit => base_values.get::<()>(()), } } diff --git a/egglog/src/prelude.rs b/egglog/src/prelude.rs index 3e8b6c5..fbe7124 100644 --- a/egglog/src/prelude.rs +++ b/egglog/src/prelude.rs @@ -860,11 +860,11 @@ impl Sort for BaseSortImpl { self.0.name() } - fn column_ty(&self, backend: &egglog_bridge::EGraph) -> ColumnTy { - ColumnTy::Base(backend.base_values().get_ty::()) + fn column_ty(&self, base_values: &BaseValues) -> ColumnTy { + ColumnTy::Base(base_values.get_ty::()) } - fn register_type(&self, backend: &mut egglog_bridge::EGraph) { + fn register_type(&self, backend: &mut dyn egglog_backend_trait::Backend) { backend.base_values_mut().register_type::(); } @@ -929,11 +929,11 @@ impl Sort for ContainerSortImpl { self.0.name() } - fn column_ty(&self, _backend: &egglog_bridge::EGraph) -> ColumnTy { + fn column_ty(&self, _base_values: &BaseValues) -> ColumnTy { ColumnTy::Id } - fn register_type(&self, backend: &mut egglog_bridge::EGraph) { + fn register_type(&self, backend: &mut dyn egglog_backend_trait::Backend) { backend.register_container_ty::(); } diff --git a/egglog/src/proofs/proof_extraction.rs b/egglog/src/proofs/proof_extraction.rs index c05c9c2..c7ccb7a 100644 --- a/egglog/src/proofs/proof_extraction.rs +++ b/egglog/src/proofs/proof_extraction.rs @@ -3,6 +3,7 @@ use crate::proofs::proof_encoding::ProofInstrumentor; use crate::proofs::proof_extractor::extract_root; use crate::proofs::proof_format::{Justification, ProofId, ProofStore, proof_store_from_term}; use crate::{RawValues, Read, ResolvedCall, TermDag}; +use egglog_backend_trait::BackendExt; use thiserror::Error; #[derive(Debug, Error)] diff --git a/egglog/src/proofs/proof_extractor.rs b/egglog/src/proofs/proof_extractor.rs index a7a5823..4a4d166 100644 --- a/egglog/src/proofs/proof_extractor.rs +++ b/egglog/src/proofs/proof_extractor.rs @@ -2,6 +2,7 @@ use crate::ast::FunctionSubtype; use crate::termdag::{TermDag, TermId}; use crate::util::{HashMap, HashSet}; use crate::{ArcSort, EGraph, Value}; +use egglog_backend_trait::BackendExt; /// Root-directed extraction for proof terms. /// diff --git a/egglog/src/proofs/proof_tests.rs b/egglog/src/proofs/proof_tests.rs index 2fd2673..6c9c5b1 100644 --- a/egglog/src/proofs/proof_tests.rs +++ b/egglog/src/proofs/proof_tests.rs @@ -188,11 +188,14 @@ mod tests { &self.name } - fn column_ty(&self, _backend: &egglog_bridge::EGraph) -> egglog_bridge::ColumnTy { + fn column_ty( + &self, + _base_values: &egglog_core_relations::BaseValues, + ) -> egglog_bridge::ColumnTy { egglog_bridge::ColumnTy::Id } - fn register_type(&self, _backend: &mut egglog_bridge::EGraph) {} + fn register_type(&self, _backend: &mut dyn egglog_backend_trait::Backend) {} fn as_arc_any( self: std::sync::Arc, diff --git a/egglog/src/scheduler.rs b/egglog/src/scheduler.rs index eb5436f..e46d312 100644 --- a/egglog/src/scheduler.rs +++ b/egglog/src/scheduler.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::sync::Mutex; use core_relations::{ExecutionState, ExternalFunction, Value}; +use egglog_backend_trait::BackendExt; use egglog_bridge::{ ColumnTy, DefaultVal, FunctionConfig, FunctionId, MergeFn, RuleId, TableAction, }; @@ -233,7 +234,12 @@ impl EGraph { record .scheduler .filter_matches(rule_id, ruleset, &mut matches); - let table_action = TableAction::new(&self.backend, rule_info.decided); + let bridge = self + .backend + .as_any() + .downcast_ref::() + .expect("scheduler match instantiation requires the reference bridge backend"); + let table_action = TableAction::new(bridge, rule_info.decided); *rule_info.matches.lock().unwrap() = matches.instantiate(state, &table_action); } }); @@ -333,7 +339,7 @@ impl SchedulerRuleInfo { .register_external_func(Box::new(CollectMatches::new(matches.clone()))); let schema = free_vars .iter() - .map(|v| v.sort.column_ty(&egraph.backend)) + .map(|v| v.sort.column_ty(egraph.backend.base_values())) .chain(std::iter::once(ColumnTy::Base(unit_type))) .collect(); let decided = egraph.backend.add_table(FunctionConfig { @@ -360,7 +366,7 @@ impl SchedulerRuleInfo { collect_matches, &entries, ColumnTy::Base(unit_type), - || "collect_matches".to_string(), + Box::new(|| "collect_matches".to_string()), ); let qrule_id = qrule_builder.build(); diff --git a/egglog/src/serialize.rs b/egglog/src/serialize.rs index 83ab829..80ce4b5 100644 --- a/egglog/src/serialize.rs +++ b/egglog/src/serialize.rs @@ -241,7 +241,7 @@ impl EGraph { // Canonicalize the value first so that we always use the canonical e-class ID let value = self .backend - .get_canon_repr(value, sort.column_ty(&self.backend)); + .get_canon_repr(value, sort.column_ty(self.backend.base_values())); assert!( !sort.name().to_string().contains('-'), "Tag cannot contain '-' when serializing" diff --git a/egglog/src/sort/fn.rs b/egglog/src/sort/fn.rs index e7f3f8e..503a743 100644 --- a/egglog/src/sort/fn.rs +++ b/egglog/src/sort/fn.rs @@ -13,6 +13,7 @@ use std::any::TypeId; use std::sync::Mutex; use crate::exec_state::Internal; +use egglog_backend_trait::BackendExt; use enum_map::EnumMap; use super::*; @@ -152,11 +153,11 @@ impl Sort for FunctionSort { &self.name } - fn column_ty(&self, _backend: &egglog_bridge::EGraph) -> ColumnTy { + fn column_ty(&self, _base_values: &BaseValues) -> ColumnTy { ColumnTy::Id } - fn register_type(&self, backend: &mut egglog_bridge::EGraph) { + fn register_type(&self, backend: &mut dyn egglog_backend_trait::Backend) { backend.register_container_ty::(); backend .base_values_mut() diff --git a/egglog/src/sort/mod.rs b/egglog/src/sort/mod.rs index 2dd56d9..3202882 100644 --- a/egglog/src/sort/mod.rs +++ b/egglog/src/sort/mod.rs @@ -52,7 +52,7 @@ pub trait Sort: Any + Send + Sync + Debug { fn name(&self) -> &str; /// Returns the backend-specific column type. See [`ColumnTy`]. - fn column_ty(&self, backend: &egglog_bridge::EGraph) -> ColumnTy; + fn column_ty(&self, base_values: &BaseValues) -> ColumnTy; /// return the inner sorts if a container sort /// remember that containers can contain containers @@ -65,7 +65,7 @@ pub trait Sort: Any + Send + Sync + Debug { } } - fn register_type(&self, backend: &mut egglog_bridge::EGraph); + fn register_type(&self, backend: &mut dyn egglog_backend_trait::Backend); fn as_arc_any(self: Arc) -> Arc; @@ -168,11 +168,11 @@ impl Sort for EqSort { &self.name } - fn column_ty(&self, _backend: &egglog_bridge::EGraph) -> ColumnTy { + fn column_ty(&self, _base_values: &BaseValues) -> ColumnTy { ColumnTy::Id } - fn register_type(&self, _backend: &mut egglog_bridge::EGraph) {} + fn register_type(&self, _backend: &mut dyn egglog_backend_trait::Backend) {} fn as_arc_any(self: Arc) -> Arc { self diff --git a/egglog/src/typechecking.rs b/egglog/src/typechecking.rs index 517082d..2ed74ec 100644 --- a/egglog/src/typechecking.rs +++ b/egglog/src/typechecking.rs @@ -256,7 +256,7 @@ impl EGraph { /// Add a user-defined sort to the e-graph. pub fn add_arcsort(&mut self, sort: ArcSort, span: Span) -> Result<(), TypeError> { - sort.register_type(&mut self.backend); + sort.register_type(self.backend.as_mut()); let name = sort.name(); match self.type_info.sorts.entry(name.to_owned()) {