From 0aa98b491398c93ceda4a249baa3c058f6a43ac4 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 13 Oct 2026 12:25:24 -0700 Subject: [PATCH] Remove the elastic upload support --- Cargo.lock | 509 +------------------------------ Cargo.toml | 1 - crates/cli/Cargo.toml | 1 - crates/cli/src/main.rs | 4 - crates/cli/src/upload.rs | 68 ----- crates/cli/tests/all/main.rs | 1 - crates/cli/tests/all/upload.rs | 56 ---- crates/upload/Cargo.toml | 14 - crates/upload/src/database.rs | 174 ----------- crates/upload/src/lib.rs | 141 --------- crates/upload/src/measurement.rs | 93 ------ 11 files changed, 4 insertions(+), 1058 deletions(-) delete mode 100644 crates/cli/src/upload.rs delete mode 100644 crates/cli/tests/all/upload.rs delete mode 100644 crates/upload/Cargo.toml delete mode 100644 crates/upload/src/database.rs delete mode 100644 crates/upload/src/lib.rs delete mode 100644 crates/upload/src/measurement.rs diff --git a/Cargo.lock b/Cargo.lock index 4a8a32df..bc7584e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,12 +157,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -204,12 +198,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.10.0" @@ -409,19 +397,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "chrono" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -494,16 +469,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1169,21 +1134,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1372,25 +1322,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -1466,40 +1397,6 @@ dependencies = [ "match_token", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humantime" version = "1.3.0" @@ -1530,43 +1427,6 @@ dependencies = [ "winapi 0.2.8", ] -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.64" @@ -2031,12 +1891,6 @@ dependencies = [ "rustix 1.1.2", ] -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minijinja" version = "2.12.0" @@ -2063,23 +1917,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "ndarray" version = "0.16.1" @@ -2237,50 +2074,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "openssl" -version = "0.10.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" -dependencies = [ - "bitflags 2.10.0", - "cfg-if 1.0.4", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.110" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "os_info" version = "3.12.0" @@ -2420,7 +2213,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" dependencies = [ - "base64 0.22.1", + "base64", "indexmap", "quick-xml", "serde", @@ -2791,46 +2584,6 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -2888,15 +2641,6 @@ dependencies = [ "rustix 1.1.2", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -2909,15 +2653,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2939,29 +2674,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "selectors" version = "0.31.0" @@ -3043,18 +2755,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -3101,7 +2801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8135a34390f8405409411deda807f6add34d1a053d4daa5e492bb21103097d63" dependencies = [ "anyhow", - "base64 0.22.1", + "base64", "opener", ] @@ -3162,7 +2862,6 @@ dependencies = [ "sightglass-data", "sightglass-fingerprint", "sightglass-recorder", - "sightglass-upload", "tempfile", "termcolor", "thiserror 1.0.69", @@ -3221,20 +2920,6 @@ dependencies = [ "wat", ] -[[package]] -name = "sightglass-upload" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "log", - "reqwest", - "serde", - "serde_json", - "sightglass-data", - "sightglass-fingerprint", -] - [[package]] name = "siphasher" version = "1.0.1" @@ -3266,16 +2951,6 @@ dependencies = [ "serde", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.1" @@ -3370,12 +3045,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "synstructure" version = "0.13.2" @@ -3402,27 +3071,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "system-interface" version = "0.27.3" @@ -3586,33 +3234,10 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "socket2 0.6.1", + "socket2", "windows-sys 0.61.2", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -3652,12 +3277,6 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.41" @@ -3689,12 +3308,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "typenum" version = "1.19.0" @@ -3780,12 +3393,6 @@ dependencies = [ "strum", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vega_lite_4" version = "0.8.1" @@ -3816,15 +3423,6 @@ dependencies = [ "libc", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3859,19 +3457,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" -dependencies = [ - "cfg-if 1.0.4", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.105" @@ -4127,7 +3712,7 @@ version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ed398988226d7aa0505ac6bb576e09532ad722d702ec4e66365d78ed695c95f" dependencies = [ - "base64 0.22.1", + "base64", "directories-next", "log", "postcard", @@ -4368,16 +3953,6 @@ dependencies = [ "wast 251.0.0", ] -[[package]] -name = "web-sys" -version = "0.3.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "web_atoms" version = "0.1.3" @@ -4551,15 +4126,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -4596,21 +4162,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -4644,12 +4195,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4662,12 +4207,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4680,12 +4219,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4710,12 +4243,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4728,12 +4255,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4746,12 +4267,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4764,12 +4279,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4794,16 +4303,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if 1.0.4", - "windows-sys 0.48.0", -] - [[package]] name = "winx" version = "0.36.4" diff --git a/Cargo.toml b/Cargo.toml index 2e194a2e..eaad7a1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ members = [ "crates/data", "crates/fingerprint", "crates/recorder", - "crates/upload", ] default-members = [ "crates/cli" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 785252da..db97765f 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -20,7 +20,6 @@ sightglass-build = { path = "../build" } sightglass-data = { path = "../data" } sightglass-fingerprint = { path = "../fingerprint" } sightglass-recorder = { path = "../recorder" } -sightglass-upload = { path = "../upload" } clap = { version = "4", features = ["derive"] } termcolor = "1" thiserror = "1.0" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 1b5aa70d..b304a559 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -6,7 +6,6 @@ mod pca_metrics; mod report; mod suite; mod summarize; -mod upload; mod validate; use anyhow::Result; @@ -21,7 +20,6 @@ use report::ReportCommand; use std::io::{self, IsTerminal}; use summarize::SummarizeCommand; use termcolor::{ColorChoice, StandardStream}; -use upload::UploadCommand; use validate::ValidateCommand; /// A stdout writer for human-readable output. Colors are only emitted when @@ -54,7 +52,6 @@ enum SightglassCommand { PcaMetrics(PcaMetricsCommand), Report(ReportCommand), Summarize(SummarizeCommand), - UploadElastic(UploadCommand), Validate(ValidateCommand), } @@ -69,7 +66,6 @@ impl SightglassCommand { SightglassCommand::PcaMetrics(metrics) => metrics.execute(), SightglassCommand::Report(report) => report.execute(), SightglassCommand::Summarize(summarize) => summarize.execute(), - SightglassCommand::UploadElastic(upload) => upload.execute(), SightglassCommand::Validate(validate) => validate.execute(), } } diff --git a/crates/cli/src/upload.rs b/crates/cli/src/upload.rs deleted file mode 100644 index 91ef5878..00000000 --- a/crates/cli/src/upload.rs +++ /dev/null @@ -1,68 +0,0 @@ -use anyhow::{Context, Result}; -use clap::Parser; -use sightglass_data::{Format, Measurement}; -use sightglass_upload::{upload, upload_package, MeasurementPackage}; -use std::{ - fs::File, - io::{self, BufReader, Read}, -}; - -/// Upload benchmark output to an ElasticSearch server; accepts raw benchmark -/// results in `stdin` (i.e., from `sightglass-cli benchmark ...`). -#[derive(Debug, Parser)] -#[command(name = "upload-elastic")] -pub struct UploadCommand { - /// The format of the input data. Either 'json' or 'csv'. - #[arg(short = 'i', long = "input-format", default_value = "json")] - input_format: Format, - - /// Path to the file that will be read from, or none to indicate stdin - /// (default). - #[arg(short = 'f', long = "input-file")] - input_file: Option, - - /// The URL of a server receiving results; this command only understands how - /// to upload results to an ElasticSearch server; e.g., - /// `http://localhost:9200`. - #[arg(default_value = "http://localhost:9200", value_name = "URL")] - server: String, - - /// Setting this flag will prevent any uploading to the server. Instead, - /// the command will emit a JSON "package" to stdout that can be used to - /// upload at a later time, see `--from-package`. - #[arg(short = 'd', long = "dry-run")] - dry_run: bool, - - /// Path to a file containing a package of measurements and fingerprint data - /// to be uploaded. If this is set, `--input-file` and `--input-format` are - /// ignored. - #[arg(short = 'p', long = "from-package")] - from_package: Option, - - /// The number of measurements to upload together; this can speed up the - /// upload. Defaults to `2000`. - #[arg(short = 'b', long = "batch-size", default_value = "2000")] - batch_size: usize, -} - -impl UploadCommand { - pub fn execute(&self) -> Result<()> { - if let Some(file) = &self.from_package { - let reader = - BufReader::new(File::open(file).context("unable to open --from-package path")?); - let package: MeasurementPackage = - serde_json::from_reader(reader).context("unable to parse --from-package JSON")?; - upload_package(&self.server, self.batch_size, self.dry_run, package) - } else { - let file: Box = if let Some(file) = self.input_file.as_ref() { - Box::new(BufReader::new( - File::open(file).context("unable to open --input-file")?, - )) - } else { - Box::new(io::stdin()) - }; - let measurements: Vec = self.input_format.read(file)?; - upload(&self.server, self.batch_size, self.dry_run, measurements) - } - } -} diff --git a/crates/cli/tests/all/main.rs b/crates/cli/tests/all/main.rs index 08f468b8..7559e73f 100644 --- a/crates/cli/tests/all/main.rs +++ b/crates/cli/tests/all/main.rs @@ -6,7 +6,6 @@ mod help; mod pca_metrics; mod report; mod summarize; -mod upload; mod util; mod validate; diff --git a/crates/cli/tests/all/upload.rs b/crates/cli/tests/all/upload.rs deleted file mode 100644 index 0307ee27..00000000 --- a/crates/cli/tests/all/upload.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Test `sightglass-cli upload`. - -// Because the `results.json` contains `*.so` suffixes for the engine, this test -// can only run where the fingerprinted engine will have a matching suffix, -// i.e., Linux. -#[cfg(target_os = "linux")] -#[test] -fn upload_dryrun() { - let assert = assert_cmd::assert::OutputAssertExt::assert( - crate::util::sightglass_cli() - .arg("upload-elastic") - .arg("--dry-run") - .arg("--input-file") - .arg("tests/results.json") - .arg("--batch-size") - .arg("200") - .env("RUST_LOG", "debug"), - ); - - // Gather up the logged output from stderr. - let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - eprintln!("=== stderr ===\n{stderr}\n==========="); - - // Gather the fingerprints of the system under test. - let engine = sightglass_fingerprint::Engine::fingerprint(crate::util::test_engine()).unwrap(); - let benchmark = - sightglass_fingerprint::Benchmark::fingerprint(crate::util::benchmark("noop")).unwrap(); - let machine = sightglass_fingerprint::Machine::fingerprint().unwrap(); - - // Check that we upload measurement records for each of the measurements in the file. - let num_uploaded_batches = stderr - .matches("Batching up 200 records to index 'measurements'") - .count(); - assert_eq!(num_uploaded_batches, 3); - - // Also, heck that we create records for the engine/machine/benchmark. - use predicates::prelude::*; - use predicates::str::*; - - assert - .stderr( - contains(format!( - r#"Creating record in 'engines' with ID Some("{}")"#, - engine.id - )) - .and(contains(format!( - r#"Creating record in 'machines' with ID Some("{}")"#, - machine.id - ))) - .and(contains(format!( - r#"Creating record in 'benchmarks' with ID Some("{}")"#, - benchmark.id - ))), - ) - .success(); -} diff --git a/crates/upload/Cargo.toml b/crates/upload/Cargo.toml deleted file mode 100644 index a945bcbd..00000000 --- a/crates/upload/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "sightglass-upload" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.0" -chrono = "0.4" -log = "0.4" -reqwest = { version = "0.11", features = ["blocking"] } -serde = { version = "1.0.118", features = ["derive", "rc"] } -serde_json = "1.0.60" -sightglass-data = { path = "../data" } -sightglass-fingerprint = { path = "../fingerprint" } diff --git a/crates/upload/src/database.rs b/crates/upload/src/database.rs deleted file mode 100644 index ad065d08..00000000 --- a/crates/upload/src/database.rs +++ /dev/null @@ -1,174 +0,0 @@ -use anyhow::{anyhow, bail, Result}; -use reqwest::blocking::Client; -use serde::{de::DeserializeOwned, Serialize}; -use serde_json::Value; -use std::collections::HashMap; - -/// A simple HTTP wrapper for communicating with an ElasticSearch database. -pub struct Database { - url: String, - dry_run: bool, -} - -impl Database { - pub fn new(url: String, dry_run: bool) -> Self { - Self { - url: url.trim_end_matches("/").to_string(), - dry_run, - } - } - - /// Use the ElasticSearch [Get - /// API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html) - /// to verify if a document exists in `index` with the given `id`. - #[allow(dead_code)] - pub fn exists(&self, index: &str, id: &str) -> Result { - let url = format!("{}/{}/_doc/{}", self.url, index, id); - if self.dry_run { - Ok(true) - } else { - let client = Client::new(); - let response = client.head(url).send()?; - Ok(response.status().is_success()) - } - } - - /// Retrieve an object from the database, if it exists. - pub fn get(&self, index: &str, id: &str) -> Result - where - T: DeserializeOwned, - { - let url = format!("{}/{}/_doc/{}", self.url, index, id); - if self.dry_run { - bail!("could not retrieve object with ID: {id}"); - } else { - let client = Client::new(); - let response = client.get(url).send()?; - let bytes = response.bytes()?; - Ok(serde_json::from_slice(bytes.as_ref())?) - } - } - - /// Create an object in the database, reusing an existing object if - /// possible. This function handles several cases: - /// 1. if the ID is not used in the database, create the object - /// 2. if the ID is used and the existing object matches `object`, simply - /// return the ID without creating a new database entry - /// 3. if the ID is used and the existing object does not match `object`, - /// append a `!` to the ID and retry (up to 5 times). - pub fn create_if_not_exists(&self, index: &str, object: &T, id: &str) -> Result - where - T: DeserializeOwned + Serialize + PartialEq, - { - let mut id = id.to_string(); - for _ in 0..NUM_RETRIES { - match self.get(index, &id) { - Ok(stored_object) => { - if object == &stored_object { - // Case #2: the same object already exists in the - // database; simply return the ID. - return Ok(id); - } else { - // Case #3: a different object exists with the same ID; - // change the ID and retry. - id.push('!'); - } - } - Err(_) => { - // Case #1: no object exists with the ID; create it. - return self.create(index, object, Some(&id)); - } - } - } - Err(anyhow!("failed to find a usable ID")) - } - - /// Use the ElasticSearch [Index - /// API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html) - /// to add a document to `index`, optionally with the given `id`. - pub fn create(&self, index: &str, object: &T, id: Option<&str>) -> Result - where - T: Serialize, - { - log::debug!("Creating record in '{index}' with ID {id:?}"); - let url = if let Some(id) = id { - format!("{}/{}/_doc/{}", self.url, index, id) - } else { - format!("{}/{}/_doc", self.url, index) - }; - - let body = serde_json::to_string(object)?; - log::trace!("Record body: {}", &body); - - if self.dry_run { - Ok(if let Some(id) = id { - id.to_string() - } else { - "dry-run-id".to_string() - }) - } else { - let client = Client::new(); - let response = client - .post(url) - .header("Content-Type", "application/json") - .body(body) - .send()?; - - let success = response.status().is_success(); - let content: HashMap = serde_json::from_slice(&response.bytes()?)?; - log::debug!("ElasticSearch response: {content:?}"); - - if success { - let id = content.get("_id").unwrap().as_str().unwrap().to_string(); - Ok(id) - } else { - bail!("Failed to create record: {content:?}") - } - } - } - - /// Use the ElasticSearch [Bulk - /// API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) - /// to add many objects to `index`. This should be significantly faster than - /// `create`. - pub fn create_batched(&self, index: &str, objects: &[T]) -> Result<()> - where - T: Serialize, - { - log::debug!("Batching up {} records to index '{}'", objects.len(), index); - let url = format!("{}/{}/_bulk", self.url, index); - - // Capture all of the records as index requests. - let mut body = vec![]; - for o in objects { - body.push("{\"index\": {}}\n".to_string()); - body.push(format!("{}\n", serde_json::to_string(o)?)); - } - let body = body.concat(); - - // Upload the entire batch request to ElasticSearch. - if self.dry_run { - log::trace!("Record body: {}", &body); - Ok(()) - } else { - let client = Client::new(); - let response = client - .post(url) - .header("Content-Type", "application/x-ndjson") - .body(body) - .send()?; - - let success = response.status().is_success(); - let content: HashMap = serde_json::from_slice(&response.bytes()?)?; - log::debug!("ElasticSearch response: {content:?}"); - - if success { - Ok(()) - } else { - bail!("Failed to batch-create records: {content:?}") - } - } - } -} - -const NUM_RETRIES: i32 = 5; diff --git a/crates/upload/src/lib.rs b/crates/upload/src/lib.rs deleted file mode 100644 index 9b602fcb..00000000 --- a/crates/upload/src/lib.rs +++ /dev/null @@ -1,141 +0,0 @@ -//! Upload Sightglass data into an ElasticSearch database. -mod database; -mod measurement; - -use crate::database::Database; -use crate::measurement::UploadMeasurement; -use anyhow::{Context, Result}; -pub use measurement::MeasurementPackage; -use sightglass_data::Measurement; -use sightglass_fingerprint::{Benchmark, Engine, Machine}; -use std::collections::{HashMap, HashSet}; -use std::io; - -/// Upload `measurements` to the `server`. -/// -/// Uploading to a database will replace several fields of the raw [Measurement] -/// with fingerprinted data from the current server; this adds useful metadata -/// to the results. E.g., the `arch` field in [Measurement] is expanded to the -/// more-complete [Machine] fingerprint. -pub fn upload( - server: &str, - batch_size: usize, - dry_run: bool, - measurements: Vec, -) -> Result<()> { - let package = package(measurements)?; - - if dry_run { - serde_json::to_writer(io::stdout(), &package) - .context("failed to write measurement package to stdout")?; - } - - upload_package(server, batch_size, dry_run, package) -} - -/// Package up the [Measurement]s alongside the fingerprint data from the -/// current system. -/// -/// The concept of "packaging up" the data allows for saving the entire -/// data collection for later upload in environments where access to the -/// database is impossible. -pub fn package(measurements: Vec) -> Result { - // De-duplicate all of the engines and benchmarks used in this result set. - let mut found_engines = HashSet::new(); - let mut found_benchmarks = HashSet::new(); - for m in measurements.iter() { - found_engines.insert(m.engine.name.clone()); - found_benchmarks.insert(m.wasm.clone()); - } - - // Map each engine path to its fingerprinted data. - let mut engines = HashMap::new(); - for engine_path in found_engines.into_iter() { - let engine = Engine::fingerprint(engine_path.as_ref())?; - engines.insert(engine_path, engine); - } - - // Map each benchmark path to its fingerprinted data. - let mut benchmarks = HashMap::new(); - for benchmark_path in found_benchmarks.into_iter() { - let benchmark = Benchmark::fingerprint(benchmark_path.as_ref())?; - benchmarks.insert(benchmark_path, benchmark); - } - - // Fingerprint the current machine. - let machine = Machine::fingerprint()?; - - // Capture the current time. - let datetime = chrono::Local::now().to_rfc3339().into(); - - Ok(MeasurementPackage { - measurements, - engines, - benchmarks, - machine, - datetime, - }) -} - -/// Upload the packaged measurements and fingerprint data to the database. -/// -/// This adds records to the database for each kind of record in the package -/// (measurements, machine, engines, benchmarks), avoiding new records for -/// fingerprinted data that is already present in the database (e.g., we do not -/// want multiple records for the same machine). If `dry_run` is set no records -/// are actually inserted, only logged. -/// -/// As described in the [tuning documentation], the optimal ElasticSearch -/// `batch_size` must be determined through experimentation. In a small -/// experiment, increasing the batch size saw diminishing returns after -/// `1000-2000`. -/// -/// [tuning documentation]: -/// https://www.elastic.co/guide/en/elasticsearch/reference/master/tune-for-indexing-speed.html#_use_bulk_requests -pub fn upload_package( - server: &str, - batch_size: usize, - dry_run: bool, - package: MeasurementPackage, -) -> Result<()> { - let database = Database::new(server.to_string(), dry_run); - - // Insert each fingerprinted version of an engine. - let mut engines = HashMap::new(); - for (path, fingerprint) in package.engines.into_iter() { - let id = database.create_if_not_exists("engines", &fingerprint, &fingerprint.id)?; - log::debug!("Mapping engine: {} -> {}", &path, &id); - engines.insert(path, id); - } - - // Insert each fingerprinted version of a benchmark. - let mut benchmarks = HashMap::new(); - for (path, fingerprint) in package.benchmarks.into_iter() { - let id = database.create_if_not_exists("benchmarks", &fingerprint, &fingerprint.id)?; - log::debug!("Mapping benchmark: {} -> {}", &path, &id); - benchmarks.insert(path, id); - } - - // Insert the machine fingerprint. - let machine = - database.create_if_not_exists("machines", &package.machine, &package.machine.id)?; - - // Insert all of the measurements. - for batch in package.measurements.chunks(batch_size) { - let batch = batch - .iter() - .map(|m| { - UploadMeasurement::map_and_convert( - &machine, - &engines, - &benchmarks, - &package.datetime, - m, - ) - }) - .collect::>(); - database.create_batched("measurements", &batch)?; - } - - Ok(()) -} diff --git a/crates/upload/src/measurement.rs b/crates/upload/src/measurement.rs deleted file mode 100644 index 12a252b7..00000000 --- a/crates/upload/src/measurement.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Provides [UploadMeasurement], a data type to convert to before uploading -//! Sightglass [Measurement] objects to a server. -use serde::{Deserialize, Serialize}; -use sightglass_data::{Measurement, Phase}; -use sightglass_fingerprint::{Benchmark, Engine, Machine}; -use std::{borrow::Cow, collections::HashMap}; - -/// A conversion of a [Measurement], with fields replaced by fingerprinting. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct UploadMeasurement<'a> { - /// The ID of the machine on which this measurement was taken; this relies - /// on `upload` to insert the data for this ID. - pub machine: Cow<'a, str>, - - /// The ID of the engine in which this measurement was taken; this relies on - /// `upload` to insert the data for this ID. - pub engine: Cow<'a, str>, - - /// The ID of the benchmark with which this measurement was taken; this - /// relies on `upload` to insert the data for this ID. - pub benchmark: Cow<'a, str>, - - /// The phase in a Wasm program's lifecycle that was measured: compilation, - /// instantiation, or execution. - pub phase: Phase, - - /// The event that was measured: micro seconds of wall time, CPU cycles - /// executed, instructions retired, cache misses, etc. - pub event: Cow<'a, str>, - - /// The event counts. - /// - /// The meaning and units depend on what the `event` is: it might be a count - /// of microseconds if the event is wall time, or it might be a count of - /// instructions if the event is instructions retired. - pub count: u64, - - /// When the measurement was collected into a package (not necessarily when - /// it was measured). - pub datetime: Cow<'a, str>, -} - -impl<'a> UploadMeasurement<'a> { - pub fn convert( - machine: &'a str, - engine: &'a str, - benchmark: &'a str, - datetime: &'a str, - measurement: &'a Measurement, - ) -> Self { - Self { - machine: Cow::Borrowed(machine), - engine: Cow::Borrowed(engine), - benchmark: Cow::Borrowed(benchmark), - phase: measurement.phase, - event: Cow::Borrowed(measurement.event.as_ref()), - count: measurement.count, - datetime: Cow::Borrowed(datetime), - } - } - - pub fn map_and_convert( - machine: &'a str, - engines: &'a HashMap, String>, - benchmarks: &'a HashMap, String>, - datetime: &'a str, - measurement: &'a Measurement, - ) -> Self { - let engine = engines - .get(measurement.engine.name.as_ref()) - .unwrap() - .as_ref(); - let benchmark = benchmarks.get(measurement.wasm.as_ref()).unwrap().as_ref(); - Self::convert(machine, engine, benchmark, datetime, measurement) - } -} - -/// This container captures all of the measurement data from a single machine in -/// a format that can be exported and later uploaded. -#[derive(Debug, Serialize, Deserialize)] -pub struct MeasurementPackage<'a> { - /// Store all the original measurements for upload. - pub measurements: Vec>, - /// Map each engine path to its fingerprinted data. - pub engines: HashMap, Engine>, - /// Map each benchmark path to its fingerprinted data. - pub benchmarks: HashMap, Benchmark>, - /// Collect the machine fingerprint. - pub machine: Machine, - /// When the measurements were collected into a package (not necessarily - /// when they were measured). - pub datetime: Cow<'a, str>, -}