diff --git a/.gitignore b/.gitignore index d181994a24d..6c609a8ff55 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,12 @@ fabric.properties # SSL artifacts data_dir/ssl_conf/ +# FTS corpora, one directory per dataset name in the test config, written by +# data_dir/latte/fts_search/generate_local_dataset.py. Gitignored because they are generated, not +# because they are temporary -- a run reads them in place. Only the rune scripts and the config +# YAMLs are tracked here. +data_dir/latte/fts_search/*/ + # Internal private repo scylla-qa-internal/ diff --git a/data_dir/latte/fts_search/fts.rn b/data_dir/latte/fts_search/fts.rn new file mode 100644 index 00000000000..d233f58c2a9 --- /dev/null +++ b/data_dir/latte/fts_search/fts.rn @@ -0,0 +1,297 @@ +//! Full-text search (BM25) workload - schema, load, build_index and search phases. +//! +//! Origin: scylladb/vector-store, latte/full-text-search/. This copy is the one test runs use -- +//! LatteStressThread resolves .rn paths inside the SCT tree and copies the whole containing +//! directory into the loader container, so the script cannot be loaded from the vector-store repo +//! directly. Mirror any change back to vector-store, and keep the phase names and -P parameter +//! names in sync with fts_test.py, which invokes them. +//! +//! Usage: +//! latte schema fts.rn +//! latte schema fts.rn -P schema_cleanup=true # full reset +//! latte schema fts.rn -P drop_index=true # drop index only +//! latte load fts.rn \ +//! --threads 1 --concurrency 10 +//! latte run -f load fts.rn -d \ +//! --threads 1 --concurrency 10 +//! latte run -f build_index fts.rn -d 1 +//! latte run -f search fts.rn -d 60s --concurrency 32 \ +//! -P 'fts_data_dir="/"' +//! +//! Schema is idempotent (CREATE TABLE IF NOT EXISTS). Use schema_cleanup=true +//! to drop the table, or drop_index=true to drop just the FTS index. + +use latte::*; + +mod metrics; + +// Schema / table +const KEYSPACE = latte::param!("keyspace", "fts_bench"); +const TABLE = latte::param!("table", "documents"); +const INDEX_NAME = latte::param!("index_name", "documents_fts_idx"); +const REPLICATION_FACTOR = latte::param!("replication_factor", 1); +const TARGET_COLUMN = latte::param!("target_column", "body"); +const INDEX_OPTIONS = latte::param!("index_options", ""); +const WITH_INDEX = latte::param!("with_index", false); +const SCHEMA_CLEANUP = latte::param!("schema_cleanup", false); +const DROP_INDEX = latte::param!("drop_index", false); +const MAX_INDEX_WAIT_SECS = latte::param!("max_index_wait_secs", 600); +const MIN_SUCCESSFUL_PROBES = latte::param!("min_successful_probes", 3); +// Number of indexed documents. When provided (> 0), indexing throughput is +// reported alongside index build time. Omit or set to 0 to skip throughput. +const DOCUMENT_COUNT = latte::param!("document_count", 0); + +// Dataset files +const FTS_DATA_DIR = latte::param!("fts_data_dir", "./"); +const DOCUMENTS_FILE = latte::param!("documents_file", "documents.tsv"); +const QUERIES_FILE = latte::param!("queries_file", "queries_natural.tsv"); +const QRELS_FILE = latte::param!("qrels_file", "qrels_natural.tsv"); + +// Search +const SEARCH_LIMIT = latte::param!("search_limit", 5); +const COMPUTE_ACCURACY = latte::param!("compute_accuracy", true); + +// Prepared statement names +const INSERT = "insert"; +const PROBE = "probe"; +const SEARCH = "search"; + +/// Create the full-text index if absent. Idempotent (IF NOT EXISTS), called +/// from `schema`, `prepare` (load context), or `prepare` (build_index context). +async fn create_index(db) { + let options_clause = if INDEX_OPTIONS == "" { "" } else { ` WITH OPTIONS = ${INDEX_OPTIONS}` }; + db.execute(`CREATE CUSTOM INDEX IF NOT EXISTS ${INDEX_NAME} ON ${KEYSPACE}.${TABLE}(${TARGET_COLUMN}) USING 'fulltext_index'${options_clause}`).await?; + Ok(()) +} + +pub async fn schema(db) { + if SCHEMA_CLEANUP { + db.execute(`DROP TABLE IF EXISTS ${KEYSPACE}.${TABLE}`).await?; + return; + } + + db.execute(`CREATE KEYSPACE IF NOT EXISTS ${KEYSPACE} WITH REPLICATION = \ + {'class': 'NetworkTopologyStrategy', 'replication_factor': '${REPLICATION_FACTOR}'}`).await?; + db.execute(`CREATE TABLE IF NOT EXISTS ${KEYSPACE}.${TABLE} (doc_id text PRIMARY KEY, ${TARGET_COLUMN} text)`).await?; + + if DROP_INDEX { + db.execute(`DROP INDEX IF EXISTS ${KEYSPACE}.${INDEX_NAME}`).await?; + } + if WITH_INDEX { + create_index(db).await?; + } +} + +fn join_path(dir, name) { + if dir == "" || dir.ends_with("/") { dir + name } else { dir + "/" + name } +} + +pub async fn prepare(db) { + let has_load = false; + let has_build_index = false; + let has_search = false; + + if !is_none(db.data.get("functions_to_invoke")) { + for item in db.data.functions_to_invoke { + let name = item.0; + if name == "load" { has_load = true; } + if name == "build_index" { has_build_index = true; } + if name == "search" { has_search = true; } + } + } else { + has_load = true; + } + + if has_load { + db.execute(`CREATE KEYSPACE IF NOT EXISTS ${KEYSPACE} WITH REPLICATION = \ + {'class': 'NetworkTopologyStrategy', 'replication_factor': '${REPLICATION_FACTOR}'}`).await?; + db.execute(`CREATE TABLE IF NOT EXISTS ${KEYSPACE}.${TABLE} (doc_id text PRIMARY KEY, ${TARGET_COLUMN} text)`).await?; + if WITH_INDEX { + create_index(db).await?; + } + + let path = join_path(FTS_DATA_DIR, DOCUMENTS_FILE); + let it = fs::read_split_lines_iter(path, ["\t"])?; + db.data.documents = []; + while let Some(entry) = it.next() { + let raw = entry?; + if !raw.is_empty() { + db.data.documents.push(#{ "id": raw[0], "body": raw[1] }); + } + } + db.load_cycle_count = db.data.documents.len(); + println!("Loaded {} documents from {}", db.data.documents.len(), path); + db.prepare(INSERT, `INSERT INTO ${KEYSPACE}.${TABLE} (doc_id, ${TARGET_COLUMN}) VALUES (:doc_id, :body)`).await?; + } + + if has_build_index { + db.execute(`DROP INDEX IF EXISTS ${KEYSPACE}.${INDEX_NAME}`).await?; + create_index(db).await?; + db.data.index_build_start = latte::now_timestamp(); + db.data.index_ready = false; + db.data.index_ready_elapsed = 0.0; + db.prepare(PROBE, `SELECT doc_id FROM ${KEYSPACE}.${TABLE} WHERE BM25(${TARGET_COLUMN}, 'probe') > 0 ORDER BY BM25(${TARGET_COLUMN}, 'probe') LIMIT 1`).await?; + db.declare_metric("index_ready_seconds", "lower"); + if DOCUMENT_COUNT > 0 { + db.declare_metric("indexing_throughput_docs_per_sec", "higher"); + } + } + + if has_search { + let queries_path = join_path(FTS_DATA_DIR, QUERIES_FILE); + let it = fs::read_split_lines_iter(queries_path, ["\t"])?; + db.data.queries = []; + while let Some(entry) = it.next() { + let raw = entry?; + if !raw.is_empty() { + db.data.queries.push(#{ "id": raw[0], "text": raw[1] }); + } + } + println!("Loaded {} queries from {}", db.data.queries.len(), queries_path); + + if COMPUTE_ACCURACY { + let qrels_path = join_path(FTS_DATA_DIR, QRELS_FILE); + let qit = fs::read_split_lines_iter(qrels_path, ["\t"])?; + db.data.qrels = []; + while let Some(entry) = qit.next() { + let raw = entry?; + if !raw.is_empty() { + db.data.qrels.push(#{ "query_id": raw[0], "doc_id": raw[1], "grade": raw[2].parse::()? }); + } + } + println!("Loaded {} qrels from {}", db.data.qrels.len(), qrels_path); + + db.data.qrels_by_query = []; + for _ in 0..db.data.queries.len() { + db.data.qrels_by_query.push([]); + } + for qrel in db.data.qrels { + for j in 0..db.data.queries.len() { + if db.data.queries[j].id == qrel.query_id { + db.data.qrels_by_query[j].push(#{ "doc_id": qrel.doc_id, "grade": qrel.grade }); + break; + } + } + } + db.data.qrels = []; + + db.declare_metric("recall", "higher"); + db.declare_metric("precision", "higher"); + db.declare_metric("mrr", "higher"); + db.declare_metric("ndcg", "higher"); + } + + println!("Search limit: {}", SEARCH_LIMIT); + + db.prepare(SEARCH, `SELECT doc_id FROM ${KEYSPACE}.${TABLE} WHERE BM25(${TARGET_COLUMN}, :q) > 0 ORDER BY BM25(${TARGET_COLUMN}, :q) LIMIT ${SEARCH_LIMIT}`).await?; + + db.set_report_field("dataset", FTS_DATA_DIR); + db.set_report_field("queries_file", QUERIES_FILE); + db.set_report_field("compute_accuracy", `${COMPUTE_ACCURACY}`); + + db.declare_metric("result_count", "higher"); + + db.data.failed_bitmap = []; + for _ in 0..db.data.queries.len() { + db.data.failed_bitmap.push(false); + } + } +} + +pub async fn load(db, i) { + let row = db.data.documents[i % db.data.documents.len()]; + db.execute_prepared(INSERT, #{ "doc_id": row.id, "body": row.body }).await?; +} + +pub async fn build_index(db, i) { + if db.data.index_ready { // done in warmup + let elapsed = db.data.index_ready_elapsed; + db.record_metric("index_ready_seconds", elapsed); + if DOCUMENT_COUNT > 0 && elapsed > 0.0 { + db.record_metric("indexing_throughput_docs_per_sec", DOCUMENT_COUNT as f64 / elapsed); + } + return Ok(()); + } + let start = db.data.index_build_start; + let max_wait = MAX_INDEX_WAIT_SECS; + let consecutive_ok = 0; + let streak_start = start; + // NOTE: this loop has no pacing of its own -- latte exposes no sleep to rune scripts. While the + // index is missing the probe fails and latte's own retry backoff paces the loop, but once + // the probes start succeeding and the streak is still short the loop spins as fast as the + // DB answers. It probes a LIMIT 1 single-term query, so the load is small, but keep in + // mind it lands on the cluster whose index build time is being measured here. Add a sleep + // binding to latte before making this loop do anything heavier. + loop { + match db.execute_prepared_with_result(PROBE, #{}).await { + Ok(_) => { + if consecutive_ok == 0 { + streak_start = latte::now_timestamp(); + } + consecutive_ok += 1; + if consecutive_ok >= MIN_SUCCESSFUL_PROBES { + let elapsed = (streak_start - start) as f64; + db.data.index_ready_elapsed = elapsed; + db.data.index_ready = true; + db.record_metric("index_ready_seconds", elapsed); + if DOCUMENT_COUNT > 0 && elapsed > 0.0 { + db.record_metric("indexing_throughput_docs_per_sec", DOCUMENT_COUNT as f64 / elapsed); + } + return Ok(()); + } + } + Err(e) => { + consecutive_ok = 0; + let err_msg = format!("{}", e); + if !(err_msg.contains("QueryRetriesExceeded") + && (latte::now_timestamp() - start) < max_wait) { + return Err(e); + } + } + } + } +} + +pub async fn search(db, i) { + if db.data.queries.len() == 0 { + return Ok(()); + } + let idx = latte::hash(i) % db.data.queries.len(); + let query = db.data.queries[idx]; + + match db.execute_prepared_with_result(SEARCH, #{ "q": query.text }).await { + Err(e) => { + if !db.data.failed_bitmap[idx] { + println!("WARNING: query failed [idx={} id=\"{}\" text=\"{}\" error={}]", + idx, query.id, query.text, e); + db.data.failed_bitmap[idx] = true; + } + return Ok(()); + } + Ok(rows) => { + let returned_ids = []; + for row in rows { + returned_ids.push(row.doc_id); + } + db.record_metric("result_count", returned_ids.len() as f64); + + if COMPUTE_ACCURACY { + let relevant_ids = []; + let relevances = []; + for qrel in db.data.qrels_by_query[idx] { + if qrel.grade > 0 { + relevant_ids.push(qrel.doc_id); + } + relevances.push(#{ "doc_id": qrel.doc_id, "grade": qrel.grade }); + } + if relevances.len() > 0 { + db.record_metric("recall", metrics::recall_at_k(returned_ids, relevant_ids, SEARCH_LIMIT)); + db.record_metric("precision", metrics::precision_at_k(returned_ids, relevant_ids, SEARCH_LIMIT)); + db.record_metric("mrr", metrics::reciprocal_rank(returned_ids, relevant_ids)); + db.record_metric("ndcg", metrics::ndcg_at_k(returned_ids, relevances, SEARCH_LIMIT)); + } + } + } + } + Ok(()) +} diff --git a/data_dir/latte/fts_search/generate_local_dataset.py b/data_dir/latte/fts_search/generate_local_dataset.py new file mode 100644 index 00000000000..1da61532514 --- /dev/null +++ b/data_dir/latte/fts_search/generate_local_dataset.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Generate the tiny local FTS datasets used by ``local_config.yaml``. + +The docker-backend FTS run reads everything straight from disk. The corpora are +generated rather than tracked in git, so run this once after a fresh clone -- the +run itself leaves them in place:: + + python3 data_dir/latte/fts_search/generate_local_dataset.py + +The output is deterministic (fixed ``SEED``), so regenerating is always safe. + +File formats consumed by ``fts.rn``: + +* ``shards/documents_NNN.tsv`` -- ``doc_idbody`` +* ``documents.tsv`` -- same, for the non-sharded path +* ``queries_.tsv`` -- ``query_idquery_text`` +* ``qrels_.tsv`` -- ``query_iddoc_idgrade`` + +Two datasets are produced, together covering every branch of +``test_fts_search``: + +``local_tiny`` + Synthetic, sharded. The vocabulary is split into buckets with different + document frequencies so BM25 has something to rank: ``COMMON`` terms appear + in most documents, ``MEDIUM`` in some, ``RARE`` in a handful. Exercises + ``_parse_shard_spec`` ranges, cumulative doc counts and ``_drop_index`` + between steps. + +``local_smoke`` + 10 hand-written documents with graded qrels, non-sharded (``documents.tsv``). + Exercises the multi-dataset loop, the ``documents_file`` branch of + ``_load_step_shards``, and qrels staging. Copied from the vector-store + repo's ``latte/full-text-search/testdata`` smoke fixture. + +Every query matches at least one document, so a zero ``result_count`` metric +means a real failure rather than a badly chosen query. +""" + +import os +import random + +SHARD_COUNT = 3 +DOCS_PER_SHARD = 300 +BODY_TOKENS = 20 +SEED = 20260729 + +COMMON = ["scylla", "cluster", "node"] +MEDIUM = ["tablets", "compaction", "keyspace", "shard"] +RARE = ["quasar", "obsidian", "zephyr"] +FILLER = [ + "data", + "write", + "read", + "latency", + "throughput", + "replica", + "token", + "range", + "memtable", + "sstable", + "cache", + "row", + "partition", + "index", + "query", + "table", + "column", + "value", + "commitlog", + "flush", +] + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATASET_DIR = os.path.join(BASE_DIR, "local_tiny") +SHARDS_DIR = os.path.join(DATASET_DIR, "shards") +SMOKE_DIR = os.path.join(BASE_DIR, "local_smoke") + +# 10-document smoke fixture with graded qrels, mirrored from the vector-store +# repo (latte/full-text-search/testdata). Inlined so this repo stays +# self-contained -- it is only ~2 KiB of text. +SMOKE_DOCUMENTS = [ + ( + "doc_001", + "The Amazon rainforest is often called the lungs of the Earth because it produces vast amounts of oxygen and absorbs carbon dioxide from the atmosphere.", + ), + ( + "doc_002", + "Renewable energy sources like solar and wind power are becoming increasingly cost-effective alternatives to fossil fuels for electricity generation.", + ), + ( + "doc_003", + "The Python programming language is widely used for data science, machine learning, and web development due to its readability and extensive library ecosystem.", + ), + ( + "doc_004", + "Mercury is the smallest planet in our solar system and orbits closest to the Sun, with surface temperatures reaching extreme highs during the day.", + ), + ( + "doc_005", + "Professional basketball players must maintain rigorous training schedules that include strength conditioning, skill drills, and strategic film study.", + ), + ( + "doc_006", + "The invention of the printing press by Johannes Gutenberg in the 15th century revolutionized the distribution of knowledge across Europe.", + ), + ( + "doc_007", + "Mediterranean cuisine emphasizes fresh vegetables, olive oil, and lean proteins, and is associated with numerous health benefits and longevity.", + ), + ( + "doc_008", + "The Great Barrier Reef off the coast of Australia is the largest coral reef system in the world and supports an extraordinary diversity of marine life.", + ), + ( + "doc_009", + "Cloud computing platforms provide on-demand access to computing resources such as virtual machines, storage, and databases over the internet.", + ), + ( + "doc_010", + "Beethoven's Symphony No. 9 is one of the most famous classical compositions and features the Ode to Joy choral finale based on Schiller's poem.", + ), +] + +SMOKE_QUERIES = [ + ("q_001", "what is the amazon rainforest known for"), + ("q_002", "renewable energy sources compared to fossil fuels"), + ("q_003", "best programming language for data science"), + ("q_004", "smallest planet in the solar system"), + ("q_005", "how do basketball players train professionally"), + ("q_006", "who invented the printing press"), + ("q_007", "benefits of mediterranean diet"), + ("q_008", "where is the great barrier reef located"), +] + +SMOKE_QRELS = [ + ("q_001", "doc_001", 3), + ("q_002", "doc_002", 3), + ("q_003", "doc_003", 3), + ("q_003", "doc_009", 1), + ("q_004", "doc_004", 3), + ("q_005", "doc_005", 3), + ("q_006", "doc_006", 3), + ("q_007", "doc_007", 3), + ("q_008", "doc_008", 3), + ("q_008", "doc_001", 1), +] + + +def _body(rng: random.Random, doc_index: int) -> str: + """Build a document body with controlled term frequencies.""" + tokens = [] + # COMMON: present in ~90% of documents. + if doc_index % 10 != 0: + tokens.append(rng.choice(COMMON)) + # MEDIUM: present in ~30%. + if doc_index % 10 < 3: + tokens.append(rng.choice(MEDIUM)) + # RARE: present in ~2%. + if doc_index % 50 == 0: + tokens.append(rng.choice(RARE)) + while len(tokens) < BODY_TOKENS: + tokens.append(rng.choice(FILLER)) + rng.shuffle(tokens) + return " ".join(tokens) + + +def write_shards(rng: random.Random) -> int: + os.makedirs(SHARDS_DIR, exist_ok=True) + total = 0 + for shard_id in range(SHARD_COUNT): + path = os.path.join(SHARDS_DIR, f"documents_{shard_id:03d}.tsv") + with open(path, "w", encoding="utf-8") as f: + for i in range(DOCS_PER_SHARD): + doc_index = shard_id * DOCS_PER_SHARD + i + f.write(f"doc_{doc_index:06d}\t{_body(rng, doc_index)}\n") + total += DOCS_PER_SHARD + print(f"wrote {path} ({DOCS_PER_SHARD} docs)") + return total + + +def write_queries() -> None: + os.makedirs(DATASET_DIR, exist_ok=True) + + # Single high-frequency terms -> large result sets, cheap to serve. + term_common = [(f"q_{i}", term) for i, term in enumerate(COMMON)] + + # Multi-term free-text queries -> more index work per query. + natural = [ + ("n_0", "scylla cluster node latency"), + ("n_1", "compaction keyspace throughput"), + ("n_2", "tablets shard partition range"), + ("n_3", "quasar obsidian zephyr"), + ] + + for name, rows in (("term_common", term_common), ("natural", natural)): + path = os.path.join(DATASET_DIR, f"queries_{name}.tsv") + with open(path, "w", encoding="utf-8") as f: + for query_id, text in rows: + f.write(f"{query_id}\t{text}\n") + print(f"wrote {path} ({len(rows)} queries)") + + +def write_smoke_dataset() -> None: + """Write the non-sharded 10-document fixture with qrels.""" + os.makedirs(SMOKE_DIR, exist_ok=True) + + # No shards/ subdir: local_config.yaml omits `shards` for this dataset, so + # _load_step_shards falls back to step["documents_file"] ("documents.tsv"). + docs_path = os.path.join(SMOKE_DIR, "documents.tsv") + with open(docs_path, "w", encoding="utf-8") as f: + for doc_id, body in SMOKE_DOCUMENTS: + f.write(f"{doc_id}\t{body}\n") + print(f"wrote {docs_path} ({len(SMOKE_DOCUMENTS)} docs)") + + queries_path = os.path.join(SMOKE_DIR, "queries_natural.tsv") + with open(queries_path, "w", encoding="utf-8") as f: + for query_id, text in SMOKE_QUERIES: + f.write(f"{query_id}\t{text}\n") + print(f"wrote {queries_path} ({len(SMOKE_QUERIES)} queries)") + + qrels_path = os.path.join(SMOKE_DIR, "qrels_natural.tsv") + with open(qrels_path, "w", encoding="utf-8") as f: + for query_id, doc_id, grade in SMOKE_QRELS: + f.write(f"{query_id}\t{doc_id}\t{grade}\n") + print(f"wrote {qrels_path} ({len(SMOKE_QRELS)} qrels)") + + +def main() -> None: + rng = random.Random(SEED) + total = write_shards(rng) + write_queries() + print(f"-> {DATASET_DIR}: {total} documents across {SHARD_COUNT} shards\n") + + write_smoke_dataset() + print(f"-> {SMOKE_DIR}: {len(SMOKE_DOCUMENTS)} documents, non-sharded, with qrels") + + +if __name__ == "__main__": + main() diff --git a/data_dir/latte/fts_search/local_config.yaml b/data_dir/latte/fts_search/local_config.yaml new file mode 100644 index 00000000000..08e24223a03 --- /dev/null +++ b/data_dir/latte/fts_search/local_config.yaml @@ -0,0 +1,69 @@ +# Tiny FTS plan for local correctness runs on the docker backend. +# +# Used by test-cases/fts-search/fts-search-test-docker.yaml. Deliberately small: +# the goal is to exercise every branch of FtsSearchTest.test_fts_search, not to +# produce meaningful performance numbers. +# +# Every shard/query file is read from data_dir/latte/fts_search/local_tiny/. The corpora are +# generated, not tracked in git, and the run leaves them in place, so this is a one-off after a +# fresh clone: +# python3 data_dir/latte/fts_search/generate_local_dataset.py +datasets: + - name: local_tiny + max_index_wait_secs: 300 + defaults: + limit: 5 + concurrency: 2 + rate: 0 + duration: 10s + expected_p99_read_ms: 10 + steps: + # Single shard: simplest load path, one rate-limited query. + - shards: [0] + queries: + - set: term_common + concurrency: 1 + rate: 50 + # Shard range: exercises _parse_shard_spec, cumulative doc counts and the + # _drop_index of the previous step's index. + - shards: [1..2] + queries: + - set: term_common + # Deliberate duplicate of the entry above -- same set and same resolved + # limit/concurrency/rate/expected_p99_read_ms, same table. Both land in the + # same step, so row_labels_for_step() must give them a ' run #1' / ' run #2' + # suffix, otherwise they collide on one Argus row. Mirrors the repeated + # term_common entries in the larger plans. + - set: term_common + - set: natural + expected_p99_read_ms: 50 + + # Second dataset: exercises the multi-dataset loop in test_fts_search + # (drop table + recreate schema per dataset). No `shards` key, so + # _load_step_shards takes the step["documents_file"] branch instead. + # qrels: true exercises qrels staging and the accuracy metrics. + - name: local_smoke + max_index_wait_secs: 300 + defaults: + limit: 5 + concurrency: 2 + rate: 0 + duration: 10s + expected_p99_read_ms: 50 + steps: + - documents_file: documents.tsv + queries: + - set: natural + qrels: true + # A rebuild on the same corpus (an empty 'shards' list, so _load_step_shards is a + # no-op, and no `queries`), exercising the "index build without a load or a + # search phase" path -- see docs/fts-search-test.md "Repeated builds on the same + # data". + - shards: [] + # Another no-load rebuild, this one repeating the first step's query set. Since + # nothing was loaded in between, all three steps report the same document count, + # so only the 'step #N' component of the row label keeps this entry off the row + # the first step already reported under. + - shards: [] + queries: + - set: natural diff --git a/data_dir/latte/fts_search/metrics.rn b/data_dir/latte/fts_search/metrics.rn new file mode 100644 index 00000000000..d98325aa6eb --- /dev/null +++ b/data_dir/latte/fts_search/metrics.rn @@ -0,0 +1,226 @@ +// IR accuracy metrics for full-text search (BM25) workloads. +// +// Origin: scylladb/vector-store, latte/full-text-search/metrics.rn. This copy is the one test +// runs use (see the header of fts.rn for why). Mirror any change back to vector-store. +// +// All metrics range from 0.0 (worst) to 1.0 (perfect). They compare what the +// search returned against the ground-truth qrels (graded relevance judgments). +// +// Example from testdata/: +// Query "where is the great barrier reef located" (q_008) +// qrels: doc_008 (grade 3), doc_001 (grade 1) +// BM25 returns: [doc_008, doc_001, ...] at k=100 + +// recall@k — "what fraction of the truly relevant docs did I find?" +// = |returned ∩ relevant| / |relevant| +// +// Counts how many of ALL relevant documents (not just the first k of the +// qrels) appear in the top-k search results. The denominator is the total +// number of known relevant docs, so recall is meaningful for queries with +// few or many relevant documents. Unlike the earlier version this does not +// build an expected set from the first min(k, |relevant|) qrels, so recall +// no longer depends on qrels file ordering. +// +// Example: k=5, relevant=[doc_001, doc_008, doc_015], returned=[doc_008, +// doc_003] → 1/3 ≈ 0.33 +pub fn recall_at_k(returned_ids, relevant_ids, k) { + if relevant_ids.len() == 0 { + return 1.0; + } + let actual_k = if returned_ids.len() < k { returned_ids.len() } else { k }; + let hits = 0; + for i in 0..actual_k { + if relevant_ids.iter().any(|r| r == returned_ids[i]) { + hits = hits + 1; + } + } + hits as f64 / relevant_ids.len() as f64 +} + +// precision@k — "of the k results I returned, how many are relevant?" +// = |returned ∩ relevant| / k +// +// Unlike recall, the denominator is always exactly k. If k=10 and only 2 of +// the 10 returned docs are relevant, precision = 0.2. +// +// Example: k=100, returned has doc_008 and doc_001 among the results +// but 0 other relevant docs → 2/100 = 0.02 +pub fn precision_at_k(returned_ids, relevant_ids, k) { + if k == 0 { + return 0.0; + } + if relevant_ids.len() == 0 { + return 0.0; + } + let actual_k = if returned_ids.len() < k { returned_ids.len() } else { k }; + let hits = 0; + for i in 0..actual_k { + if relevant_ids.iter().any(|r| r == returned_ids[i]) { + hits = hits + 1; + } + } + hits as f64 / k as f64 +} + +// reciprocal rank (MRR per query) — "how early did the first relevant doc +// appear?" +// = 1 / rank_of_first_relevant +// +// If the first relevant doc is at position 1, RR=1.0; at position 3, RR=0.33; +// if none are found, RR=0.0. +// +// Example: returned=[doc_003, doc_008, doc_001] for q_008 +// first relevant is doc_008 at rank 2 → 1/2 = 0.5 +pub fn reciprocal_rank(returned_ids, relevant_ids) { + if relevant_ids.len() == 0 { + return 0.0; + } + let rank = 0; + for i in 0..returned_ids.len() { + if relevant_ids.iter().any(|r| r == returned_ids[i]) { + rank = i + 1; + break; + } + } + if rank == 0 { + return 0.0; + } + 1.0 / rank as f64 +} + +// Look up the relevance grade of a returned doc from the qrels. +// Returns 0.0 if the doc has no judgment for the current query. +fn relevance_for_id(relevances, doc_id) { + for rel in relevances { + if rel.doc_id == doc_id { + return rel.grade as f64; + } + } + 0.0 +} + +// Precomputed 1/log2(i+2) discount factors for nDCG positions 0..199. +// (Rune has powf but no log2, so these are computed offline in Python.) +fn ndcg_discount(i) { + let table = [ + 1.00000000_f64, 0.63092975_f64, 0.50000000_f64, 0.43067656_f64, + 0.38685281_f64, 0.35620719_f64, 0.33333333_f64, 0.31546488_f64, + 0.30103000_f64, 0.28906483_f64, 0.27894295_f64, 0.27023815_f64, + 0.26264954_f64, 0.25595802_f64, 0.25000000_f64, 0.24465054_f64, + 0.23981247_f64, 0.23540891_f64, 0.23137821_f64, 0.22767025_f64, + 0.22424382_f64, 0.22106473_f64, 0.21810429_f64, 0.21533828_f64, + 0.21274605_f64, 0.21030992_f64, 0.20801460_f64, 0.20584683_f64, + 0.20379505_f64, 0.20184909_f64, 0.20000000_f64, 0.19823986_f64, + 0.19656163_f64, 0.19495902_f64, 0.19342640_f64, 0.19195872_f64, + 0.19055141_f64, 0.18920036_f64, 0.18790182_f64, 0.18665241_f64, + 0.18544902_f64, 0.18428883_f64, 0.18316925_f64, 0.18208790_f64, + 0.18104260_f64, 0.18003133_f64, 0.17905223_f64, 0.17810359_f64, + 0.17718382_f64, 0.17629143_f64, 0.17542506_f64, 0.17458343_f64, + 0.17376534_f64, 0.17296969_f64, 0.17219543_f64, 0.17144160_f64, + 0.17070728_f64, 0.16999162_f64, 0.16929381_f64, 0.16861310_f64, + 0.16794878_f64, 0.16730018_f64, 0.16666667_f64, 0.16604765_f64, + 0.16544255_f64, 0.16485086_f64, 0.16427205_f64, 0.16370566_f64, + 0.16315122_f64, 0.16260831_f64, 0.16207652_f64, 0.16155547_f64, + 0.16104477_f64, 0.16054409_f64, 0.16005307_f64, 0.15957142_f64, + 0.15909881_f64, 0.15863496_f64, 0.15817959_f64, 0.15773244_f64, + 0.15729325_f64, 0.15686177_f64, 0.15643779_f64, 0.15602107_f64, + 0.15561139_f64, 0.15520856_f64, 0.15481238_f64, 0.15442266_f64, + 0.15403922_f64, 0.15366189_f64, 0.15329049_f64, 0.15292487_f64, + 0.15256487_f64, 0.15221035_f64, 0.15186115_f64, 0.15151715_f64, + 0.15117821_f64, 0.15084420_f64, 0.15051500_f64, 0.15019048_f64, + 0.14987054_f64, 0.14955506_f64, 0.14924394_f64, 0.14893706_f64, + 0.14863434_f64, 0.14833567_f64, 0.14804096_f64, 0.14775011_f64, + 0.14746305_f64, 0.14717969_f64, 0.14689994_f64, 0.14662372_f64, + 0.14635096_f64, 0.14608158_f64, 0.14581551_f64, 0.14555268_f64, + 0.14529302_f64, 0.14503647_f64, 0.14478295_f64, 0.14453241_f64, + 0.14428479_f64, 0.14404003_f64, 0.14379807_f64, 0.14355885_f64, + 0.14332233_f64, 0.14308844_f64, 0.14285714_f64, 0.14262838_f64, + 0.14240211_f64, 0.14217828_f64, 0.14195685_f64, 0.14173777_f64, + 0.14152100_f64, 0.14130649_f64, 0.14109421_f64, 0.14088412_f64, + 0.14067617_f64, 0.14047033_f64, 0.14026656_f64, 0.14006482_f64, + 0.13986509_f64, 0.13966731_f64, 0.13947147_f64, 0.13927753_f64, + 0.13908545_f64, 0.13889521_f64, 0.13870677_f64, 0.13852011_f64, + 0.13833519_f64, 0.13815199_f64, 0.13797047_f64, 0.13779062_f64, + 0.13761241_f64, 0.13743580_f64, 0.13726078_f64, 0.13708732_f64, + 0.13691539_f64, 0.13674498_f64, 0.13657605_f64, 0.13640859_f64, + 0.13624257_f64, 0.13607797_f64, 0.13591477_f64, 0.13575295_f64, + 0.13559250_f64, 0.13543338_f64, 0.13527558_f64, 0.13511908_f64, + 0.13496386_f64, 0.13480991_f64, 0.13465720_f64, 0.13450572_f64, + 0.13435545_f64, 0.13420637_f64, 0.13405847_f64, 0.13391173_f64, + 0.13376614_f64, 0.13362168_f64, 0.13347832_f64, 0.13333607_f64, + 0.13319491_f64, 0.13305481_f64, 0.13291577_f64, 0.13277777_f64, + 0.13264079_f64, 0.13250483_f64, 0.13236988_f64, 0.13223591_f64, + 0.13210292_f64, 0.13197089_f64, 0.13183981_f64, 0.13170967_f64, + 0.13158046_f64, 0.13145216_f64, 0.13132477_f64, 0.13119827_f64, + 0.13107265_f64, 0.13094791_f64, 0.13082402_f64, 0.13070099_f64, + ]; + if i < table.len() { table[i] } else { 1.0 / (i + 2) as f64 } +} + +// nDCG@k (normalized Discounted Cumulative Gain) — "how good is the ranking +// compared to the ideal ranking?" +// +// Uses graded relevance (not just binary relevant/not). A high-grade doc +// (grade=3) contributes more gain than a low-grade doc (grade=1). Documents +// ranked lower are discounted by 1/log2(rank+1). +// +// DCG@k = sum_{i=1}^{k} (2^{rel_i} - 1) / log2(i + 1) +// IDCG@k = same with relevance grades sorted descending (ideal order) +// nDCG@k = DCG / IDCG +// +// nDCG=1.0 means perfect ranking. nDCG=0.0 means no relevant docs found. +// +// Example: q_008, relevances=[(doc_008, grade 3), (doc_001, grade 1)] +// Returned: [doc_008, doc_001] +// DCG = (2^3-1)/1 + (2^1-1)/1.585 = 7 + 0.63 = 7.63 +// IDCG = (2^3-1)/1 + (2^1-1)/1.585 = 7 + 0.63 = 7.63 +// nDCG = 7.63/7.63 = 1.0 (perfect, doc_008 was ranked first) +// +// If returned: [doc_001, doc_008] (wrong order) +// DCG = (2^1-1)/1 + (2^3-1)/1.585 = 1 + 4.42 = 5.42 +// nDCG = 5.42/7.63 = 0.71 +pub fn ndcg_at_k(returned_ids, relevances, k) { + let actual_k = if returned_ids.len() < k { returned_ids.len() } else { k }; + if actual_k == 0 { + return 0.0; + } + + let dcg = 0.0; + for i in 0..actual_k { + let rel = relevance_for_id(relevances, returned_ids[i]); + dcg = dcg + ((2.0_f64).powf(rel) - 1.0) * ndcg_discount(i); + } + + let grades = []; + for rel in relevances { + if rel.grade > 0 { + grades.push(rel.grade as f64); + } + } + + if grades.len() == 0 { + return 0.0; + } + + let n = grades.len(); + for i in 0..n { + for j in (i + 1)..n { + if grades[j] > grades[i] { + let tmp = grades[i]; + grades[i] = grades[j]; + grades[j] = tmp; + } + } + } + + let ideal_len = if grades.len() < k { grades.len() } else { k }; + let idcg = 0.0; + for i in 0..ideal_len { + idcg = idcg + ((2.0_f64).powf(grades[i]) - 1.0) * ndcg_discount(i); + } + + if idcg == 0.0 { + return 0.0; + } + dcg / idcg +} diff --git a/defaults/test_default.yaml b/defaults/test_default.yaml index 3dbfd2232a9..5c11542d459 100644 --- a/defaults/test_default.yaml +++ b/defaults/test_default.yaml @@ -309,6 +309,7 @@ zero_token_instance_type_db: '' use_zero_nodes: false latte_schema_parameters: {} +search_test_config: null perf_stress_keyspace: null perf_stress_table: null workload_name: '' diff --git a/docs/configuration_options.md b/docs/configuration_options.md index 871d9b27f3f..d2304179dc7 100644 --- a/docs/configuration_options.md +++ b/docs/configuration_options.md @@ -4676,6 +4676,15 @@ Optional. Allows to pass through custom rune script parameters to the 'latte sch **type:** dict | YAML/JSON string → dict +## **search_test_config** / SCT_SEARCH_TEST_CONFIG + +Search test definition (datasets, shards, query sets and their runtime settings).
Accepts an absolute path, or one relative to the SCT root, e.g. data_dir/latte/fts_search/plan.yaml.
Required by a search test: it is the definition of what to run, so there is nothing to fall back on.
Per-query-set rate, duration and index-wait values live inside this file, not in SCT params. + +**default:** N/A + +**type:** str (appendable) + + ## **perf_stress_keyspace** / SCT_PERF_STRESS_KEYSPACE Keyspace name used in performance gradual throughput tests.
Required for all stress tools (cassandra-stress, scylla-bench, cql-stress-cassandra-stress, latte).
For latte, if not set, falls back to the 'keyspace' key in latte_schema_parameters. diff --git a/docs/fts-search-test.md b/docs/fts-search-test.md new file mode 100644 index 00000000000..34ad296bf5f --- /dev/null +++ b/docs/fts-search-test.md @@ -0,0 +1,373 @@ +# Running the FTS (BM25 full-text search) performance test + +`fts_test.FtsSearchTest.test_fts_search` drives a full-text-search benchmark: +load documents → build a `fulltext_index` → run query sets → report latency and +index-build metrics to Argus. Index build time is read from vector-store's own "full scan" +log lines (see "Index build timing" below). + +The flow is not specific to full text. It lives in `search_perf_test.py` and is shared with the +other benchmarks of a vector-store-served index; `fts_test.py` is the full-text half — the rune +script to run, the vocabulary to report in, and the names to report under, all in one +`SearchWorkload`. Index build timing and the Argus build table live in +`sdcm/utils/vector_store_index.py`, index and readiness polling in `sdcm/utils/vector_store_client.py`. + +So far there is one way to run it: + +| | Backend | Purpose | Cost | Wall clock | +|---|---|---|---|---| +| [Local](#1-local-correctness-run-docker-backend) | `docker` | Verify the test *orchestration* is correct | none | ~5 min | + +The local run produces meaningless numbers — 1 shard of 300 synthetic documents on a +containerised Scylla. Use it to check that shard staging, index building, metric +parsing and the Argus tables all work. A run on real hardware against real corpora comes +separately. + +> **Note:** minicloud cannot run this test. It emulates only `i4i.large` and +> `n2-highmem-2`, the vector-store AMI is arm64 (minicloud is KVM on x86), and the +> integration is still an unmerged draft. See `docs/plans/minicloud-local-testing.md`. + +--- + +## 1. Local correctness run (docker backend) + +### One-time setup + +**Build a vector-store image.** The docker backend takes a prebuilt image only — it +has no way to build vector-store itself. Build the commit you care +about from the vector-store repo (it needs `fulltext_index` support, i.e. +`crates/vector-store/src/fts_index/`): + +```bash +cd /vector-store +docker build -t local/vector-store:fts . +``` + +If you build a commit other than `local/vector-store:fts`, update +`vector_store_docker_image` / `vector_store_version` in +`test-cases/fts-search/fts-search-test-docker.yaml`. + +**Optional — silence a spurious ERROR event.** Scylla in an unprivileged container +logs `Perf-based stall detector creation failed (EACCESS) ... to enable kernel +backtraces`. SCT's BACKTRACE pattern `^(?!.*audit:).*backtrace` matches the word +"backtraces" and promotes it to an ERROR event, which makes `finalize_teardown()` +fail the run even when the test body passed. To get a fully green run: + +```bash +sudo sysctl -w kernel.perf_event_paranoid=1 # host-wide; 2 is the Fedora default +``` + +Without this you get `1 passed, 1 error`, where the error is teardown-only. + +### Every run + +```bash +cd /scylla-cluster-tests + +unset DOCKER_HOST # SCT's docker backend needs a real dockerd, not podman +export JOB_NAME=local_run # see note below + +# Generate the corpora if you have not already -- they are not tracked in git. The run reads +# them in place and leaves them alone, so this is a one-off. +python3 data_dir/latte/fts_search/generate_local_dataset.py + +./docker/env/hydra.sh run-test fts_test.FtsSearchTest.test_fts_search \ + --backend docker \ + --config test-cases/fts-search/fts-search-test-docker.yaml +``` + +**Why `JOB_NAME=local_run`.** Hydra forwards `-e JOB_NAME="${JOB_NAME}"`. With the +variable unset on the host that arrives inside the container as an *empty string* +rather than unset, which defeats the `local_run` default in `get_job_name()` +(`sdcm/utils/ci_tools.py`). SCT then treats the run as CI and connects to the real +Argus, creating a junk run there. Setting it explicitly keeps Argus in replay-only +mode: every submission is written to `argus_replay_log_*.jsonl` in the run's log +directory and nothing is posted. + +Drop that line if you *want* to see the tables render in Argus for real — which is +the strongest check of any change to the `expected_p99_read_ms` table split. + +**If the tables do not show up in Argus,** check both gates before suspecting the test — +either one silently downgrades the whole run to replay-only, and neither fails loudly: + +```python +# sdcm/test_config.py, TestConfig.init_argus_client() +if params.get("enable_argus") and get_job_name() != "local_run": +``` + +So `unset JOB_NAME SCT_ENABLE_ARGUS` for a run whose results you want posted. Both are +*environment* state, so they outlive the run that needed them — a `JOB_NAME=local_run` +exported for one run is still set for the next one in the same shell, and that run will +silently post nothing either. To confirm which gate you tripped: + +```bash +grep -m1 -oE "'(enable_argus|job_name)': [^,]*" ~/sct-results/latest/argus.log +grep -c "replay-log-only" ~/sct-results/latest/argus.log # >0 means nothing was posted +``` + +The results themselves are still in `argus_replay_log_*.jsonl` (one record per +`submit_results` call), but there is no replay CLI to push them after the fact — a run whose +numbers you actually need has to be repeated. + +### Use hydra, not a bare `sct.py`, on Fedora + +Running SCT outside the hydra container **fails on a Fedora host**: + +```bash +# Does NOT work on Fedora 43. +export SCT_CLUSTER_BACKEND=docker +export SCT_CONFIG_FILES=test-cases/fts-search/fts-search-test-docker.yaml +uv run sct.py run-test fts_test.FtsSearchTest.test_fts_search +``` + +`DockerLoaderNode` runs on the host via `LOCALRUNNER` (`sdcm/cluster_docker.py`), so +`SetUp()` installs packages onto the host. The Fedora entry in `sdcm/utils/distro.py` +recognises only `34`/`35`/`36`, so on Fedora 43 the distro resolves to `UNKNOWN`, +`is_rhel_like` is `False`, and `install_package` falls through to the apt branch: + +``` +Distro: missed key for ('fedora', '43') +Unable to detect Linux distribution name +sudo apt-get ... install -y tar -> sudo: apt-get: command not found +``` + +Inside hydra the loader's "host" is the hydra container, which SCT recognises as +Debian-like, so `apt-get` is correct there. Adding `43` to that Fedora entry would make the +non-hydra path work. + +Also do not substitute a bare `pytest fts_test.py::...` on Python 3.14: SCT's +`EventsDevice` is not picklable and 3.14 defaults to the `forkserver` start method, +so the event system dies with `TypeError: cannot pickle 'weakref.ReferenceType'`. +`ensure_start_method()` (which forces `fork`) is called from `sct.py` and +`unit_tests/conftest.py`, but not from the repo-root `conftest.py`. + +### What to check afterwards + +The numbers say nothing here, so "did it work?" has to be answered from the result tables. Logs +land in `~/sct-results//`: + +```bash +D=$(ls -dt ~/sct-results/*/ | head -1) + +# Index build times come from vector-store's own 'full scan' log lines, not from anything the +# stress tool prints -- see "Index build timing" below. +grep -E "Index build time \(vector-store full scan\)" $D/sct.log + +# Cross-check against the source those numbers are read from. Each reported build should match a +# 'starting'/'finished' pair for the same index (note the lower-cased index name). +grep -E "(starting|finished) full scan on" $D/*vs-set*/*/system.log + +# The expected_p99_read_ms split: local_config.yaml exercises fts_search_p99_10ms (term_common) +# and fts_search_p99_50ms (natural). +python3 -c "import json; print(list(json.load(open('$D/latency_results.json'))))" + +# Argus submissions (replay-only mode). Expect at least these tables: +# FTS Index Build Time +# read - fts_search_p99_10ms - latencies +# read - fts_search_p99_50ms - latencies +grep -o "read - fts_search_p99_[a-z0-9_]* - latencies\|FTS Index Build Time" \ + $D/argus_replay_log_*.jsonl | sort -u +``` + +The shape to expect — one build row per step, one latency table per distinct +`expected_p99_read_ms`, a `step #N` in every query row, and a `run #N` only where a step repeats +a query configuration verbatim: + +``` +Index build time (vector-store full scan): (local_tiny | 300 docs | build #1) +Index build time (vector-store full scan): (local_tiny | 900 docs | build #2) +Index build time (vector-store full scan): (local_smoke | 10 docs | build #1) +Index build time (vector-store full scan): (local_smoke | 10 docs | build #2) +Index build time (vector-store full scan): (local_smoke | 10 docs | build #3) + +FTS Index Build Time rows: 5 (one per step, 'build #N'-suffixed) +read - fts_search_p99_10ms - lat. rows: local_tiny | 300 docs | step #1 | term_common, + local_tiny | 900 docs | step #2 | term_common | + limit=5 concurrency=2 rate=0 run #1, + ... the same with run #2 +read - fts_search_p99_50ms - lat. rows: local_tiny | 900 docs | step #2 | natural, + local_smoke | 10 docs | step #1 | natural, + local_smoke | 10 docs | step #3 | natural +``` + +local_smoke's second and third builds have no load — they rebuild the index on the corpus the +first step already loaded, so their build time reflects only the index rebuild, not the load (see +"Repeated builds on the same data" below). `build #2` also has no queries at all, and `build #3` +repeats `build #1`'s query set — which is why its query rows differ from step #1's only in the +`step #N`. + +The pieces that can be checked without a cluster already are, so a failure here is more likely to be +the orchestration than the plumbing underneath it: + +```bash +# corpus staging and the load, against a real ScyllaDB +pytest -m integration unit_tests/integration/test_search_perf_test.py +# index-status polling and the build-time log parsing, against a real vector-store +pytest -m integration unit_tests/integration/test_vector_store.py +``` + +### Expected noise (all harmless) + +- `Dashboard with title 'Overview' was not found`, then a connection failure to + alertmanager on `127.0.0.1:9093` — log collection looking for Grafana dashboards + the docker monitor does not have. Costs ~3 minutes at the end of the run. +- `nodetool_*_failure_*.log`, `StorageConfigurationCollector: FAIL`, + `TCPConnectionsCollector: FAIL` — scylla-doctor probes that do not apply in a + container. + +### Cleanup + +The test case sets `execute_post_behavior: true` with `post_behavior_*: keep-on-failure`, so a +passing run removes its containers and a failing one leaves them up for inspection. Without that, +`clean_resources()` logs "Resources will continue to run" and every run leaks its db and +vector-store containers — the default is `false` because in Jenkins a separate stage does the +destroying, and a local run has no such stage. + +Containers are labelled with the run's TestId, so a failed or interrupted run cleans up with: + +```bash +docker ps -a --filter label=TestId= -q | xargs -r docker rm -f +``` + +To sweep every SCT container regardless of run (careful — this takes the monitoring stack too): + +```bash +docker ps -a --filter label=TestId -q | xargs -r docker rm -f +``` + +--- + +## Notes on the test config format + +The dataset/query plan is a separate YAML from the SCT test case: + +`search_test_config` accepts two forms (`resolve_test_config_path()` in `search_perf_test.py`): + +| Value | Resolved as | +|---|---| +| `/abs/local/path` | used as-is | +| `data_dir/latte/fts_search/plan.yaml` | relative to the SCT root | + +The option is not full-text specific: the plan format belongs to the shared flow, so a vector-search +test case will name its own plan through the same option. + +It has no default — which datasets, shards and query sets to run *is* the definition of the test, +so a test case has to name a plan. The plans live in the repo, next to the rune scripts they +drive: + +| Plan | Used by | Size | +|---|---|---| +| `local_config.yaml` | the docker test case | two tiny generated corpora, read from disk | + +Rate, duration and index-wait are per-query-set values inside the plan YAML — there are no SCT +params for them. Per dataset: + +| Key | Default | Bounds | +|---|---|---| +| `max_index_wait_secs` | 1800 | the rune script's own budget for probing the index until it answers, the index-build phase timeout, and how long SCT waits for a dropped index to disappear | +| `max_shard_load_secs` | 3600 | the load phase timeout, **per shard** — shards load one at a time | + +### Every query needs an expected latency + +Every query entry must resolve an `expected_p99_read_ms`, either on the entry itself or the +dataset's `defaults` — there is no SCT-side default or hardcoded threshold, and missing it on both +raises `ValueError` as soon as the plan is read: + +```yaml +defaults: + expected_p99_read_ms: 10 # inherited by every query in the dataset unless overridden + +steps: + - queries: + - set: term_common # uses the default: 10ms + - set: natural + expected_p99_read_ms: 50 # overridden: 50ms +``` + +The value selects the Argus table — `read - fts_search_p99_{value}ms - latencies` +(`_cycle_name()` in `search_perf_test.py`) — and becomes that table's `P99 read` validation rule. +It is a property of the table, not a column, so queries resolving to the same value share a table. +What varies per row is reported as columns instead: `limit`, `concurrency`, `rate` and a +`query_example` (the first line of the query set's `.tsv`). + +The whole plan is resolved up front, right after it is read (`validate_plan_queries()` in +`search_perf_test.py`), so a typo fails before a step spends tens of minutes loading shards. Note +that a query's `duration` has to be latte's *time* form (`60s`, `5m`, `1h`), not its request-count +form — `get_timeout_from_stress_cmd()` only parses the former, and a duration it cannot read +silently gives the search phase the whole `test_duration` as its timeout. + +### Repeated query configs get a disambiguating suffix + +Argus row labels are built from dataset / document count / step ordinal / query set +(`row_labels_for_step()` in `search_perf_test.py`); the query parameters are columns, not part of +the label. Two entries for the same set within one step therefore collide, and a colliding label +gets a suffix naming its query configuration — plus a positional `run #N` when the entries agree +on every parameter too. A label that does not collide is left byte-identical, so Argus history +stays continuous. So: + +```yaml +- set: term_common + concurrency: 1 + rate: 50 +- set: term_common + limit: 100 +- set: term_common # identical to the next one +- set: term_common +``` + +produces + +``` +ds | N docs | step #1 | term_common | limit=5 concurrency=1 rate=50 +ds | N docs | step #1 | term_common | limit=100 concurrency=32 rate=0 +ds | N docs | step #1 | term_common | limit=5 concurrency=32 rate=0 run #1 +ds | N docs | step #1 | term_common | limit=5 concurrency=32 rate=0 run #2 +``` + +`local_config.yaml` exercises it deliberately — `local_tiny`'s second step repeats +`set: term_common` with identical parameters. Covered by `unit_tests/unit/test_search_perf_test.py`. + +### Repeated builds on the same data + +A step with an **empty** `shards` list loads nothing — `_load_step_shards` returns 0 — so it only +drops the previous index and rebuilds one on the corpus already in the table. Useful for sampling +index-build-time variance in isolation from load time: + +(An *absent* `shards` key is a different thing: it falls back to the step's `documents_file`, i.e. +a single unsharded corpus. See `local_smoke` in `local_config.yaml`.) + +```yaml +steps: + - shards: [0] # load ~100k documents (cold build, includes any one-off warmup cost) + - shards: [] # rebuild on the same 100k documents, warm + - shards: [] + - shards: [] +``` + +Each build still gets its own Argus row (`{dataset} | {doc_count} docs | build #{N}`, one per +step regardless of whether it loaded anything), so repeats do not collide -- see `local_smoke` +in `local_config.yaml`, which does a build followed by a warm rebuild on the same corpus. + +Queries are optional too (`step.get("queries", [])` defaults to none), so a step can be build-only. + +### Index build timing + +Index build time and indexing throughput are measured by SCT from **vector-store's own log** +(`sdcm.utils.vector_store_index.parse_full_scan_seconds`), not from anything the `build_index` +stress command prints. vector-store brackets an index's initial table scan with two INFO lines +carrying microsecond timestamps, and that scan *is* the build: + +``` +2026-07-30T23:05:37.908018Z INFO ... db_index{fts_bench.fts_idx_10m_20tok_0}: starting full scan on fts_bench.fts_idx_10m_20tok_0 +2026-07-30T23:06:43.914698Z INFO ... db_index{fts_bench.fts_idx_10m_20tok_0}: finished full scan on fts_bench.fts_idx_10m_20tok_0 +``` + +The stress tool still owns the DDL and still decides when the index is usable (its `build_index` +probes BM25 until it answers); SCT only reads the log afterwards, via `BaseNode.system_log` — which +resolves to `hosts//messages.log` under `logs_transport: vector` and to +`/system.log` otherwise. + +**Case folding.** Scylla folds unquoted identifiers, so `CREATE CUSTOM INDEX fts_idx_10M_20tok_0` is +`fts_idx_10m_20tok_0` everywhere downstream — that is the name in `system_schema.indexes`, the key +vector-store uses, and the key in the log lines above. Anything SCT sends to or matches against the +vector-store API goes through `sdcm.utils.vector_store_index.index_key`; querying with the unfolded +name 404s forever. diff --git a/fts_test.py b/fts_test.py new file mode 100644 index 00000000000..449de8e6d62 --- /dev/null +++ b/fts_test.py @@ -0,0 +1,92 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Full-Text Search (BM25) performance test. + +The flow -- plan, datasets, cumulative shard steps, index build timing, query sets -- lives in +search_perf_test.py and is shared with the other search benchmarks. This module is the full-text +half of it: the rune script to run, the vocabulary to report in, and the names to report under. + +Driven by a YAML plan (the `search_test_config` param) naming the datasets, the shards and the +query configurations to test. See docs/fts-search-test.md. + +Results go to Argus: one row per index build in the "FTS Index Build Time" table, and one +latency/throughput row per query configuration in "read - fts_search_p99_{expected_p99_read_ms}ms - +latencies". +""" + +from argus.client.generic_result import StaticGenericResultTable + +from search_perf_test import LatteScriptParams, SearchPerformanceTest, SearchWorkload +from sdcm.utils.vector_store_index import index_build_columns + +FTS_BASE_DIR = "data_dir/latte/fts_search" + +# The column of the index build table counting what was indexed. Named once: it goes both into the +# table definition and into every row submitted, and Argus keys the table's history by it. +FTS_BUILD_COUNT_COLUMN = "document_count" + + +class FtsIndexBuildResult(StaticGenericResultTable): + class Meta: + name = "FTS Index Build Time" + description = "Full-text search index build time and throughput" + Columns = index_build_columns(FTS_BUILD_COUNT_COLUMN, "docs") + + +FTS_WORKLOAD = SearchWorkload( + name="fts_search", + base_dir=FTS_BASE_DIR, + script=f"{FTS_BASE_DIR}/fts.rn", + hdr_tag="fn--search", + item_noun="docs", + index_prefix="fts_idx", + default_keyspace="fts_bench", + remote_root="/tmp/fts", + latency_legend="FTS BM25 full-text search query latency.", + build_result_table=FtsIndexBuildResult, + build_count_column=FTS_BUILD_COUNT_COLUMN, + # The names fts.rn uses. It is mirrored from scylladb/vector-store, so they are its to choose -- + # test_fts_test.py checks that each one is still a parameter of the script. + params=LatteScriptParams( + dataset_dir="fts_data_dir", + records_file="documents_file", + record_count="document_count", + queries_file="queries_file", + qrels_file="qrels_file", + search_limit="search_limit", + compute_accuracy="compute_accuracy", + index_name="index_name", + max_index_wait="max_index_wait_secs", + min_probes="min_successful_probes", + schema_cleanup="schema_cleanup", + drop_index="drop_index", + ), + step_records_file_key="documents_file", + default_records_file="documents.tsv", + default_shard_suffix="documents_{:03d}.tsv", +) + + +class FtsSearchTest(SearchPerformanceTest): + """FTS (Full-Text Search / BM25) performance test. + + Runs multi-dataset, multi-step FTS benchmarks from a YAML plan in the repo (see + 'resolve_test_config_path'): per-shard loading, index building, query execution and Argus + reporting all come from 'SearchPerformanceTest'. + """ + + WORKLOAD = FTS_WORKLOAD + + def test_fts_search(self): + self.run_search_benchmark() diff --git a/sdcm/argus_results.py b/sdcm/argus_results.py index 142d218180e..6e341906ab8 100644 --- a/sdcm/argus_results.py +++ b/sdcm/argus_results.py @@ -315,6 +315,8 @@ def send_result_to_argus( # noqa: PLR0914 result: dict, start_time: float = 0, error_thresholds: dict = None, + extra_columns: list[ColumnMetadata] = None, + extra_values: dict = None, ): """Sends results to Argus service. @@ -330,8 +332,18 @@ def send_result_to_argus( # noqa: PLR0914 - Reactor stalls table is registered when relevant SCT events occured (result['reactor_stalls_stats']) during the measured time range. + + :param extra_columns: caller-supplied columns appended to the main result table only (not the + summary table, since these are per-row metadata rather than something to aggregate). Used + together with 'extra_values' below; both are None for every existing caller. + :param extra_values: {column name: value} written once per row alongside the usual latency + cells, for the columns named in 'extra_columns'. Only emitted for callers whose result has a + single HDR tag (the same branch that writes 'duration'/'start time'): with several tags there + is one row per tag and no per-tag value to write, so they are skipped rather than repeated. """ result_table, result_table_summary = workload_to_table[workload](), workload_to_table[workload]() + if extra_columns: + result_table.columns = [*result_table.columns, *extra_columns] if type(cycle) is int: cycle = f"Cycle #{cycle}" result_table.name = f"{workload} - {name} - latencies" @@ -392,6 +404,9 @@ def send_result_to_argus( # noqa: PLR0914 result_table.add_result(column="Overview", row=row_name, value=overview_screenshot[0], status=Status.UNSET) if qa_screenshot: result_table.add_result(column="QA dashboard", row=row_name, value=qa_screenshot[0], status=Status.UNSET) + if extra_values: + for column_name, value in extra_values.items(): + result_table.add_result(column=column_name, row=row_name, value=value, status=Status.UNSET) if hdr_summary_len > 2: result_table_summary.add_result( diff --git a/sdcm/sct_config.py b/sdcm/sct_config.py index e7591e71ba5..3976175b6f1 100644 --- a/sdcm/sct_config.py +++ b/sdcm/sct_config.py @@ -2426,6 +2426,16 @@ class SCTConfiguration(BaseModel): perf_stress_keyspace/perf_stress_table are not set. For example, {'keyspace': 'test_keyspace', 'table': 'test_table'}""", ) + + # Search (full-text, vector) performance test config options + + search_test_config: String = SctField( + description="""Search test definition (datasets, shards, query sets and their runtime settings). + Accepts an absolute path, or one relative to the SCT root, e.g. data_dir/latte/fts_search/plan.yaml. + Required by a search test: it is the definition of what to run, so there is nothing to fall back on. + Per-query-set rate, duration and index-wait values live inside this file, not in SCT params.""", + ) + perf_stress_keyspace: String = SctField( description="""Keyspace name used in performance gradual throughput tests. Required for all stress tools (cassandra-stress, scylla-bench, cql-stress-cassandra-stress, latte). diff --git a/sdcm/stress/latte_thread.py b/sdcm/stress/latte_thread.py index 2a3c31f5090..9f3be556f57 100644 --- a/sdcm/stress/latte_thread.py +++ b/sdcm/stress/latte_thread.py @@ -75,7 +75,7 @@ def get_latte_operation_type(stress_cmd): counter_read = True elif re.findall(r"(?:^|_)(write|insert|update|delete)(?:_|$)", fn): write_found = True - elif re.findall(r"(?:^|_)(read|select|get|count)(?:_|$)", fn): + elif re.findall(r"(?:^|_)(read|select|get|count|search)(?:_|$)", fn): read_found = True else: return "user" @@ -95,6 +95,20 @@ class LatteStressThread(DockerBasedStressThread): DOCKER_IMAGE_PARAM_NAME = "stress_image.latte" SCHEMA_CMD_CALL_COUNTER = {} + def __init__(self, *args, extra_files_to_stage=None, **kwargs): + """*extra_files_to_stage* is a list of '(local_path, remote_path)' pairs to copy into the + loader container before the run. + + 'build_stress_cmd' already ships every top-level file of the rune script's directory, which + covers whatever a script needs on every invocation. This is for data that only one + invocation needs: a dataset shard, say, too big to keep in the tree and gone again before + the next command runs. Nothing needs cleaning up afterwards -- the 'RemoteDocker' command + runner of '_run_stress' is created and destroyed per latte invocation, so the container is + the cleanup context. + """ + super().__init__(*args, **kwargs) + self.extra_files_to_stage: list[tuple[str, str]] = extra_files_to_stage or [] + def set_stress_operation(self, stress_cmd): return get_latte_operation_type(self.stress_cmd) @@ -156,6 +170,9 @@ def build_stress_cmd(self, cmd_runner, loader, hosts): if not cmd_runner.run(f"test -f {remote_path}", ignore_status=True, verbose=False).ok: cmd_runner.send_files(str(src_file), remote_path) + for local_path, remote_path in self.extra_files_to_stage: + cmd_runner.send_files(local_path, remote_path, verbose=False) + ssl_config = self._build_ssl_config(cmd_runner, loader) auth_config = "" diff --git a/sdcm/tester.py b/sdcm/tester.py index d3b4abc96e5..26a8b7b30f4 100644 --- a/sdcm/tester.py +++ b/sdcm/tester.py @@ -3194,6 +3194,7 @@ def run_latte_thread( stats_aggregate_cmds=True, stop_test_on_failure=True, node_list=None, + extra_files_to_stage=None, **_, ): if duration: @@ -3216,6 +3217,7 @@ def run_latte_thread( round_robin=round_robin, stop_test_on_failure=stop_test_on_failure, params=self.params, + extra_files_to_stage=extra_files_to_stage, ).run() def run_hydra_kcl_thread( diff --git a/sdcm/utils/decorators.py b/sdcm/utils/decorators.py index 733d0b57d24..db5e02860cf 100644 --- a/sdcm/utils/decorators.py +++ b/sdcm/utils/decorators.py @@ -192,6 +192,8 @@ def latency_calculator_decorator( # noqa: PLR0915 cycle_name: Optional[str] = None, workload_type: Optional[str] = None, row_name: Optional[str] = None, + error_thresholds: Optional[dict] = None, + extra_columns: Optional[list] = None, ): """ Gets the start time, end time and then calculates the latency based on function 'calculate_latency'. @@ -203,6 +205,14 @@ def latency_calculator_decorator( # noqa: PLR0915 are absent from the results, the rest is collected as usual. :param func: Remote method to run. + :param error_thresholds: overrides the 'latency_decorator_error_thresholds' test param for this + call, e.g. when a caller computes its own validation rule instead of relying on a value + configured for the whole test (see fts_test.py, whose expected latency comes from its plan). + :param extra_columns: additional 'argus.client.generic_result.ColumnMetadata' appended to the + result table's schema. The decorated function's return value may then include an + 'extra_values' dict of {column name: value}, submitted once per row alongside the usual + latency cells. Ignored (as is 'extra_values') unless both are supplied, so existing callers + are unaffected. :return: Wrapped method. """ # calling this import here, because of circular import @@ -315,7 +325,12 @@ def wrapped(*args, **kwargs): # noqa: PLR0912, PLR0914 result["cycle_hdr_throughput"] = round(hdr_throughput) result["reactor_stalls_stats"] = reactor_stall_stats LOGGER.debug("Reactor stalls stats: %s", reactor_stall_stats) - error_thresholds = tester.params.get("latency_decorator_error_thresholds") + thresholds_config = ( + error_thresholds + if error_thresholds is not None + else tester.params.get("latency_decorator_error_thresholds") + ) + extra_values = res.get("extra_values") if isinstance(res, dict) else None if "steady" in func_name.lower(): if "Steady State" not in latency_results: latency_results["Steady State"] = result @@ -327,7 +342,9 @@ def wrapped(*args, **kwargs): # noqa: PLR0912, PLR0914 cycle=row_name or 0, result=result, start_time=start, - error_thresholds=error_thresholds, + error_thresholds=thresholds_config, + extra_columns=extra_columns, + extra_values=extra_values, ) else: latency_results[func_name]["cycles"].append(result) @@ -341,7 +358,9 @@ def wrapped(*args, **kwargs): # noqa: PLR0912, PLR0914 cycle=row_name or len(latency_results[func_name]["cycles"]), result=result, start_time=start, - error_thresholds=error_thresholds, + error_thresholds=thresholds_config, + extra_columns=extra_columns, + extra_values=extra_values, ) LOGGER.debug("Saved in Argus") diff --git a/sdcm/utils/hdrhistogram.py b/sdcm/utils/hdrhistogram.py index ec7d2c304af..0bddb5007df 100644 --- a/sdcm/utils/hdrhistogram.py +++ b/sdcm/utils/hdrhistogram.py @@ -427,7 +427,7 @@ def _get_workload_type_by_hdr_tag(self, hdr_tag): LOGGER.debug(f"Checking hdr_tag {hdr_tag} for workload type detection") if any(w_word in hdr_tag for w_word in ("write", "insert", "update", "delete")): return "WRITE" - elif any(r_word in hdr_tag for r_word in ("read", "select", "get", "count", "scan")): + elif any(r_word in hdr_tag for r_word in ("read", "select", "get", "count", "scan", "search")): return "READ" elif self.stress_operation in ("WRITE", "READ"): # branch for the scylla-bench case with its 'co-fixed' and 'raw' tags diff --git a/sdcm/utils/vector_store_client.py b/sdcm/utils/vector_store_client.py index 7914a276128..31e77921a96 100644 --- a/sdcm/utils/vector_store_client.py +++ b/sdcm/utils/vector_store_client.py @@ -38,7 +38,7 @@ def request(self, method: str, endpoint: str, **kwargs) -> requests.Response: response.raise_for_status() return response - def get_status(self) -> dict: + def get_status(self) -> str: """Get Vector Store operational status""" return self.request("GET", "/api/v1/status").json() @@ -67,17 +67,41 @@ def get_index_status(self, keyspace: str, index: str) -> dict: """Get status and vector count for a specific index""" return self.request("GET", f"/api/v1/indexes/{keyspace}/{index}/status").json() + def get_index_status_or_none(self, keyspace: str, index: str) -> dict | None: + """Get status and count for an index, or None if it has not been discovered yet. + + A newly created index is not immediately visible to Vector Store -- the endpoint 404s + until it is -- so callers that need to poll through that discovery gap (e.g. measuring + index build time) should use this instead of 'get_index_status'. + """ + try: + return self.get_index_status(keyspace, index) + except requests.exceptions.HTTPError as exc: + if exc.response is not None and exc.response.status_code == 404: + return None + raise + def get_index_count(self, keyspace: str, index: str) -> int: """Get number of embeddings in a vector index""" return self.get_index_status(keyspace, index)["count"] - def wait_for_ready(self, timeout: int = 300, check_interval: int = 5) -> bool: - """Wait for Vector Store to become ready (to have status SERVING or INDEXING_VECTORS)""" + def wait_for_ready( + self, + timeout: int = 300, + check_interval: float = 5, + required_statuses: tuple[str, ...] = ("SERVING", "BOOTSTRAPPING"), + ) -> bool: + """Wait for Vector Store to report one of *required_statuses* + + The default accepts BOOTSTRAPPING, which is what cluster readiness means here: the service + answers and is catching up. A caller that needs it to be actually serving before it measures + anything -- index build time, query latency -- should pass ('SERVING',) instead. + """ end_time = time.time() + timeout while time.time() < end_time: try: status = self.get_status() - if status in ("SERVING", "BOOTSTRAPPING"): + if status in required_statuses: LOGGER.info("Vector Store is ready (status: %s)", status) return True except Exception: # noqa: BLE001 @@ -88,5 +112,34 @@ def wait_for_ready(self, timeout: int = 300, check_interval: int = 5) -> bool: LOGGER.error("Vector Store did not become ready within %s seconds", timeout) return False + def wait_for_index_absent(self, keyspace: str, index: str, timeout: float, check_interval: float = 1.0): + """Wait until an index is no longer discoverable, or raise once *timeout* has passed + + Vector Store's view of an index is asynchronous in both directions: it takes time to + discover a fresh index, and time to forget a dropped one. A caller that recreates an index + needs this wait, or 'CREATE CUSTOM INDEX' can race a drop that ScyllaDB already considers + done but Vector Store has not caught up with yet. + + A failed request counts as "still there" rather than as a disappearance: mistaking a network + hiccup for a completed drop would let the next build start too early. Both names must + already be case-folded the way ScyllaDB folds unquoted identifiers -- querying with the + unfolded name 404s forever, which would read as "already dropped". + """ + deadline = time.monotonic() + timeout + while True: + try: + status = self.get_index_status_or_none(keyspace, index) + except Exception as exc: # noqa: BLE001 + LOGGER.debug("Index status check for '%s.%s' failed: %s", keyspace, index, exc) + status = {"status": f"request failed: {exc}"} + if status is None: + return + if time.monotonic() >= deadline: + raise RuntimeError( + f"Index '{keyspace}.{index}' was not dropped within {timeout}s " + f"(last status: {status.get('status')})" + ) + time.sleep(check_interval) + def close(self): self.session.close() diff --git a/sdcm/utils/vector_store_index.py b/sdcm/utils/vector_store_index.py new file mode 100644 index 00000000000..5de7552c80b --- /dev/null +++ b/sdcm/utils/vector_store_index.py @@ -0,0 +1,160 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Measuring how long a vector-store index took to build, and reporting it to Argus. + +Applies to any index vector-store serves -- a full-text ('fulltext_index') or a vector +('vector_index') one -- because the measurement comes from vector-store's own log rather than from +whatever built the index. +""" + +import logging +import re +import time +from datetime import datetime + +from argus.client.generic_result import ColumnMetadata, ResultType, StaticGenericResultTable, Status + +from sdcm.argus_results import submit_results_to_argus + +LOGGER = logging.getLogger(__name__) + +# How often the vector-store log is re-read while waiting for a finished build's 'full scan' lines. +# Each attempt reads the log from the start, and on the aws backend that is the whole node's +# 'messages.log', which keeps growing over a multi-hour run -- so poll it sparingly. The lines are +# normally there on the first read; this interval only matters when the log shipper lags. +FULL_SCAN_LOG_POLL_INTERVAL_SECS = 2.0 +# How long to wait for a finished build's 'full scan' log lines to reach the runner. They are written +# on the vector-store node and forwarded asynchronously, so they can lag the build by a moment; this +# only bounds that lag, it is not a wait for the build itself. +DEFAULT_FULL_SCAN_LOG_WAIT = 120 + +# Vector-store logs both ends of an index's initial table scan at INFO with a microsecond tracing +# timestamp ("starting/finished full scan on .", see its db_index.rs). That scan is +# the index build, so those two lines are the most direct measurement available -- and the most +# precise. The alternatives are both worse: latte's clock ('latte::now_timestamp()') has +# whole-second resolution, and vector-store's index-status endpoint serves a snapshot refreshed on a +# ~1s ticker, so polling it observes each edge up to a second late. +# +# Two line formats have to be handled, because 'BaseNode.system_log' resolves elsewhere depending on +# 'logs_transport': +# docker /system.log the raw tracing line +# aws /hosts//messages.log the same line behind a log-shipper prefix +# Only the tracing timestamp is followed by 'Z', which is what tells the two apart. +_FULL_SCAN_RE = re.compile( + r"(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?)Z" + r".*?\b(?Pstarting|finished) full scan on (?P\S+)" +) + + +def index_key(keyspace: str, index_name: str) -> str: + """Return the '.' key vector-store uses for an index, case-folded. + + Vector-store keys an index by the name it reads out of 'system_schema.indexes', and Scylla + case-folds unquoted identifiers -- so 'CREATE CUSTOM INDEX fts_idx_10M_20tok_0' is known + downstream as 'fts_idx_10m_20tok_0'. Anything SCT sends to, or matches against, the vector-store + API has to be folded the same way, or it silently never matches. + """ + return f"{keyspace}.{index_name}".lower() + + +def parse_full_scan_seconds(log_path: str, key: str) -> float | None: + """Return the duration of *key*'s initial full scan, read from a vector-store log. + + Returns None when the log holds no complete scan for that index -- no matching lines yet (the + log is shipped asynchronously, so a caller should retry), or a start without a finish. Index + names are unique per build in the search performance tests, so the first complete start->finish + pair is the right one. + """ + key = key.lower() + started = None + try: + with open(log_path, encoding="utf-8", errors="replace") as log_file: + for line in log_file: + match = _FULL_SCAN_RE.search(line) + if not match or match.group("key").lower() != key: + continue + timestamp = datetime.fromisoformat(match.group("ts")) + if match.group("event") == "starting": + started = timestamp + elif started is not None: + return (timestamp - started).total_seconds() + except OSError: + return None + return None + + +def wait_for_index_build_seconds( + log_path: str, + keyspace: str, + index_name: str, + timeout: float = DEFAULT_FULL_SCAN_LOG_WAIT, + poll_interval: float = FULL_SCAN_LOG_POLL_INTERVAL_SECS, +) -> float | None: + """Return how long an index's initial full scan took, from the vector-store node's log. + + Meant to be called once the build is over, so both log lines already exist on the node -- but + they reach the runner asynchronously (a tailing thread on docker, a log shipper on aws), so give + them a bounded window to arrive rather than reading once. + + Normally the first read finds them; the retries are for the shipper lagging, and are paced by + *poll_interval* because each one re-reads the whole log. Returns None if the lines never turn + up, which a caller should report as a missing measurement rather than as a failed build. + """ + key = index_key(keyspace, index_name) + deadline = time.monotonic() + timeout + while True: + build_seconds = parse_full_scan_seconds(log_path, key) + if build_seconds is not None: + return build_seconds + if time.monotonic() >= deadline: + LOGGER.warning("No complete 'full scan' log pair for '%s' in %s after %ss", key, log_path, timeout) + return None + time.sleep(poll_interval) + + +def index_build_columns(count_column: str, count_unit: str) -> list[ColumnMetadata]: + """Columns of an index-build Argus table: how long the build took, over how much data. + + *count_column* names what was indexed, in the workload's own vocabulary ('document_count', + 'vector_count'), because that name is what the table's history is keyed by. The table itself is + declared per workload, so its name and description stay next to the test that owns them. + """ + return [ + ColumnMetadata(name="build_time", unit="s", type=ResultType.FLOAT, higher_is_better=False), + ColumnMetadata(name=count_column, unit=count_unit, type=ResultType.INTEGER, higher_is_better=False), + ColumnMetadata( + name="indexing_throughput", unit=f"{count_unit}/s", type=ResultType.FLOAT, higher_is_better=True + ), + ] + + +def send_index_build_result( + argus_client, + result_table: StaticGenericResultTable, + count_column: str, + build_time: float, + count: int, + row_key: str, +): + """Submit one index build row to Argus. + + Argus merges rows into the table it already has under this name, so submitting a single row + per index build is enough -- the same way 'send_result_to_argus' reports one latency row per + 'row_name' (see performance_regression_alternator_test.py). + """ + throughput = round(count / build_time, 1) if build_time > 0 and count > 0 else 0.0 + result_table.add_result(column="build_time", row=row_key, value=build_time, status=Status.UNSET) + result_table.add_result(column=count_column, row=row_key, value=count, status=Status.UNSET) + result_table.add_result(column="indexing_throughput", row=row_key, value=throughput, status=Status.UNSET) + submit_results_to_argus(argus_client, result_table) diff --git a/search_perf_test.py b/search_perf_test.py new file mode 100644 index 00000000000..800a3bd27b1 --- /dev/null +++ b/search_perf_test.py @@ -0,0 +1,864 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Workload-agnostic search performance flow, shared by the search benchmarks. + +The flow is the same whichever index is under test -- full-text (BM25 over documents) or vector +(ANN over embeddings) -- because vector-store serves both and latte drives both: + + a YAML plan names datasets; a dataset is a sequence of steps; a step loads more shards on top + of the previous ones, rebuilds the index and runs its query sets, so one dataset yields results + at several corpus sizes. Index build time comes from vector-store's own log (see + sdcm.utils.vector_store_index), query latency from latte's HDR output, and every row is streamed + to Argus as it is produced. + +What differs per workload is the rune script, the vocabulary and the names things are reported +under. All of it is declared in a 'SearchWorkload', which a subclass points 'WORKLOAD' at; the +subclass then only owns its Argus table and its 'test_*' entry point. See fts_test.py for a worked +example, and docs/fts-search-test.md for the plan format. + +Every query entry must resolve an ``expected_p99_read_ms`` (on the query itself or the dataset's +``defaults``): it both groups results into the Argus table for that latency expectation and becomes +the table's validation rule, so there is no SCT-side hardcoded threshold or label. +""" + +import math +import os +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import PurePosixPath + +import yaml # type: ignore + +from performance_regression_test import PerformanceRegressionTest +from sdcm import sct_abs_path +from sdcm.sct_events.database import DatabaseLogEvent +from sdcm.sct_events.filters import DbEventsFilter +from sdcm.utils.decorators import latency_calculator_decorator +from sdcm.utils.vector_store_client import VectorStoreClient +from sdcm.utils.vector_store_index import send_index_build_result, wait_for_index_build_seconds + +from argus.client.generic_result import ColumnMetadata, ResultType + +# The SCT param naming the plan to run. One option for every search workload rather than one per +# test: the plan format is the flow's, not any single workload's, so a vector-search test reuses it. +TEST_CONFIG_PARAM = "search_test_config" + +DEFAULT_MAX_INDEX_WAIT = 1800 +DEFAULT_RATE = 0 +DEFAULT_DURATION = "60s" +DEFAULT_LIMIT = 5 +DEFAULT_CONCURRENCY = 2 + +# How often SCT re-checks the vector-store state it is waiting on over the API: the node reporting +# SERVING, and a dropped index disappearing. Neither is on a measured path -- the build time comes +# from log timestamps, not from when a poll happened to notice -- so this only bounds how long the +# test lingers past the event, and a shorter interval would only add requests. +VECTOR_STORE_STATUS_POLL_INTERVAL_SECS = 1.0 +# How long a run waits for the vector-store node to report SERVING before the first index build. +# Cluster init already waits for it (see 'VectorStoreClusterMixin.wait_for_init'), so this is a +# second, narrower gate against the specific "actually serving, not just bootstrapping" state index +# timing needs, not a full readiness wait. +DEFAULT_VECTOR_STORE_SERVING_WAIT = 300 + +# Timeouts for the latte commands that carry no '--duration', in seconds. They are needed because +# without one 'run_latte_thread' falls back to the whole 'test_duration', which turns any +# hung phase into a run that sits on its cluster until the Jenkins job times out. +DEFAULT_SCHEMA_TIMEOUT = 600 +# Per shard, not per step -- shards are loaded one latte invocation at a time. +DEFAULT_MAX_SHARD_LOAD = 3600 +# Added to the index build phase's own wait before it becomes SCT's outer timeout, so the script's +# give-up path always wins the tie -- see '_build_index'. +INDEX_BUILD_TIMEOUT_GRACE_SECS = 120 + +# Names taken from the plan which end up in a CQL identifier, in a shell command or in a file path. +# Validated once on the way in instead of being quoted differently at each of those three places. +SAFE_NAME_RE = re.compile(r"[A-Za-z0-9_]+") +SAFE_DATA_FILE_RE = re.compile(r"[A-Za-z0-9_.\-/]+") +# A latte '--duration': digits and one of its unit suffixes, the form 'get_timeout_from_stress_cmd' +# parses a phase timeout out of. +SAFE_DURATION_RE = re.compile(r"\d+[hms]") + +# Cap on the 'query_example' cell below. Argus' TEXT column takes whatever it is given, and the value +# comes from a corpus the plan names rather than from anything validated here. +QUERY_EXAMPLE_MAX_CHARS = 256 + +# Columns added to the search latency tables alongside the usual latency/throughput ones, so that +# the query configuration and an example query are visible per row instead of folded into an +# increasingly long row label (see 'row_labels_for_step'). +SEARCH_EXTRA_COLUMNS = [ + ColumnMetadata(name="limit", unit="", type=ResultType.INTEGER), + ColumnMetadata(name="concurrency", unit="", type=ResultType.INTEGER), + ColumnMetadata(name="rate", unit="ops/s", type=ResultType.INTEGER), + ColumnMetadata(name="query_example", unit="", type=ResultType.TEXT), +] + + +@dataclass(frozen=True) +class LatteScriptParams: + """The '-P' parameter names of a workload's rune script. + + Each rune script keeps its own vocabulary -- the full-text one talks about documents -- and the + scripts are mirrored from scylladb/vector-store rather than owned here, so the flow maps its + neutral notion of a record onto whatever the script calls it. Declared per workload instead of + fixed, so adding a workload never means renaming parameters in a script that lives elsewhere. + + Every field is a name the script must accept; nothing else belongs here, so a test can check the + whole descriptor against the script it names. + """ + + dataset_dir: str + records_file: str + record_count: str + queries_file: str + qrels_file: str + search_limit: str + compute_accuracy: str + index_name: str + max_index_wait: str + min_probes: str + schema_cleanup: str + drop_index: str + + +@dataclass(frozen=True) +class SearchWorkload: + """Everything that makes a search benchmark specific to one kind of index.""" + + name: str # 'fts_search' -- prefixes the Argus cycle name + base_dir: str # holds the rune script, the tracked plans and the local datasets + script: str # the rune script latte runs + hdr_tag: str # HDR tag its search function emits, e.g. 'fn--search' + item_noun: str # 'docs' -- the unit row labels count in + index_prefix: str # 'fts_idx' -- prefixes every index this test builds + default_keyspace: str + remote_root: str # where datasets are staged inside the loader container + latency_legend: str # first sentence of the Argus latency table description + build_result_table: type # StaticGenericResultTable subclass for index build rows + build_count_column: str # its column counting what was indexed, e.g. 'document_count' + params: LatteScriptParams + # Plan keys and defaults naming the data files of a step, in the same vocabulary as the script. + step_records_file_key: str + default_records_file: str + default_shard_suffix: str + + +def _local_path(workload: SearchWorkload, *parts: str) -> str: + """Return the absolute path to a file inside the workload's data directory.""" + return sct_abs_path(os.path.join(workload.base_dir, *parts)) + + +def _timeout_minutes(seconds: int) -> int: + """Convert a phase budget in seconds to the whole minutes 'run_latte_thread' takes.""" + return max(1, math.ceil(seconds / 60)) + + +def _checked_name(name: str, kind: str) -> str: + """Validate a plan-supplied name used as a CQL identifier and as a path component.""" + if not isinstance(name, str) or not SAFE_NAME_RE.fullmatch(name): + raise ValueError(f"Invalid {kind} {name!r}: expected only letters, digits and underscores") + return name + + +def _checked_data_file(name: str, kind: str) -> str: + """Validate a plan-supplied, dataset-relative data file name. + + The name is interpolated into the shell command that stages the file into the loader container + and joined onto the local dataset directory, so reject both shell metacharacters and anything + that escapes the dataset directory. + """ + if not isinstance(name, str) or not SAFE_DATA_FILE_RE.fullmatch(name): + raise ValueError(f"Invalid {kind} {name!r}: expected only letters, digits, '.', '-', '_' and '/'") + parts = PurePosixPath(name).parts + if name.startswith("/") or ".." in parts: + raise ValueError(f"Invalid {kind} {name!r}: must be a relative path inside the dataset directory") + return name + + +def _checked_int(value, kind: str, minimum: int = 0) -> int: + """Validate a plan-supplied number that is interpolated into a latte command line.""" + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"Invalid {kind} {value!r}: expected an integer >= {minimum}") + return value + + +def _checked_positive_float(value, kind: str) -> float: + """Validate a plan-supplied latency expectation. + + Same bool-before-number care as '_checked_int', plus the values 'float()' accepts and nothing + downstream can use: a non-positive expectation is a validation rule no run can satisfy, and + 'nan'/'inf' reach '_format_ms' and end up in an Argus table name. + """ + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + raise ValueError(f"Invalid {kind} {value!r}: expected a finite number > 0") + return float(value) + + +def _checked_duration(value, kind: str) -> str: + """Validate a plan-supplied latte duration, e.g. '60s'. + + The time form specifically, not latte's request-count form: 'get_timeout_from_stress_cmd' only + recognises '', and a duration it cannot parse silently gives the phase the whole + 'test_duration' as its timeout (see the '--duration' NOTE in '_run_search'). + """ + if not isinstance(value, str) or not SAFE_DURATION_RE.fullmatch(value): + raise ValueError(f"Invalid {kind} {value!r}: expected a latte duration like '60s', '5m' or '1h'") + return value + + +def _count_tsv_lines(path: str) -> int: + """Count non-empty lines in a TSV file.""" + count = 0 + with open(path) as f: + for line in f: + if line.strip(): + count += 1 + return count + + +def _first_query_example(local_ds_dir: str, queries_file: str) -> str: + """Return the text of the first query in a 'queries_.tsv' file, or "" if unavailable. + + Rows are '\\t' (see data_dir/latte/fts_search/generate_local_dataset.py); only the + text is kept, since it is what makes an Argus row readable at a glance. Read once, on demand, + rather than cached: the file is small and this only runs once per query-set/step. + + Truncated, because the length is not ours to bound: a plan can point 'base_url' at any corpus, + natural-language query sets run long, and a row without the tab separator yields the whole line. + The marker keeps a cut example from reading as a complete one. + """ + path = os.path.join(local_ds_dir, queries_file) + try: + with open(path, encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + parts = stripped.split("\t", 1) + text = parts[1] if len(parts) > 1 else parts[0] + if len(text) <= QUERY_EXAMPLE_MAX_CHARS: + return text + return text[: QUERY_EXAMPLE_MAX_CHARS - 3] + "..." + except FileNotFoundError: + pass + return "" + + +def _parse_shard_spec(shards: list) -> list[int]: + """Normalize shard spec into a flat list of ints. + + Accepts ints and 'start..end' range strings, e.g. [0..9, 10, 11..99]. Anything else raises: + silently dropping it would load a smaller corpus than the plan asked for, and the run would + report perfectly plausible numbers for the wrong record count. + """ + result: list[int] = [] + for item in shards: + # NOTE: bool before int -- 'isinstance(True, int)' holds, and YAML turns an unquoted + # 'yes'/'no'/'on'/'off' into a bool, so those would otherwise pass as shard 0/1. + if isinstance(item, bool) or not isinstance(item, (int, str)): + raise ValueError(f"Invalid shard spec entry {item!r}: expected an int or a 'start..end' string") + if isinstance(item, int): + result.append(item) + else: + m = re.fullmatch(r"(\d+)\.\.(\d+)", item) + if not m: + raise ValueError(f"Invalid shard range: {item!r}") + start, end = int(m.group(1)), int(m.group(2)) + # A descending range is the same silent under-load as a dropped entry: it expands to + # nothing, and the run reports plausible numbers for a corpus it never loaded. + if end < start: + raise ValueError(f"Invalid shard range: {item!r} ends before it starts") + result.extend(range(start, end + 1)) + # A repeated shard is the mirror image of a dropped one, and just as quiet: loading it twice + # upserts the same document ids, so the table gains nothing while 'record_count' counts both + # invocations -- inflating the reported corpus size and the indexing throughput derived from it. + # The set comparison is the fast path; the quadratic 'count()' only runs once a duplicate is + # known to exist, on the way to raising. + if len(result) != len(set(result)): + duplicates = sorted({shard for shard in result if result.count(shard) > 1}) + raise ValueError(f"Invalid shard spec {shards!r}: shard(s) {duplicates} appear more than once") + return result + + +def resolve_test_config_path(config: str) -> str: + """Resolve the plan param to a local file. + + Two accepted forms, so that pointing a run at a different plan is a single overridable + param (SCT_SEARCH_TEST_CONFIG): + + /a/local/path used as-is + data_dir/latte/fts_search/plan.yaml relative to the SCT root, like every other file a + test case names -- see 'scylla_d_overrides_files' and + the '.rn' paths inside latte stress commands + """ + if os.path.isabs(config): + return config + return sct_abs_path(config) + + +# --------------------------------------------------------------------------- +# Query discovery +# --------------------------------------------------------------------------- + + +def _query_params(query: dict, defaults: dict) -> tuple: + """Resolve (limit, concurrency, rate) for a query entry, applying dataset defaults. + + Checked, not just read: all three are interpolated into the latte command line, the same reason + '_checked_name' exists for the plan's names. + """ + return ( + _checked_int(query.get("limit", defaults.get("limit", DEFAULT_LIMIT)), "query limit", minimum=1), + _checked_int( + query.get("concurrency", defaults.get("concurrency", DEFAULT_CONCURRENCY)), "query concurrency", minimum=1 + ), + # 0 is 'unthrottled' -- '_run_search' drops '--rate' entirely for it. + _checked_int(query.get("rate", defaults.get("rate", DEFAULT_RATE)), "query rate"), + ) + + +def _query_duration(query: dict, defaults: dict) -> str: + """Resolve the latte '--duration' for a query entry, applying dataset defaults.""" + return _checked_duration(query.get("duration", defaults.get("duration", DEFAULT_DURATION)), "query duration") + + +def _expected_p99_read_ms(query: dict, defaults: dict) -> float: + """Resolve the expected P99 read latency (ms) for a query entry. + + Required -- on the query itself or the dataset's 'defaults' -- rather than defaulted by SCT: + it both groups the query into an Argus table (see '_cycle_name') and becomes that table's + validation rule, so the expectation lives entirely in the plan, not as a hardcoded SCT value. + """ + expected = query.get("expected_p99_read_ms", defaults.get("expected_p99_read_ms")) + if expected is None: + raise ValueError( + f"Query set {query.get('set')!r} has no 'expected_p99_read_ms' " + f"(set it on the query entry or the dataset's 'defaults')" + ) + return _checked_positive_float(expected, f"expected_p99_read_ms for query set {query.get('set')!r}") + + +def _format_ms(value: float) -> str: + """Render an expected-latency value for use in an Argus table/cycle name. + + NOTE: not '{:g}' -- that switches to scientific notation past six digits, so a plan asking for + 10000000 would name its table 'p99_1e+07ms'. + """ + return f"{value:f}".rstrip("0").rstrip(".").replace(".", "_") + + +def _cycle_name(workload: SearchWorkload, expected_p99_read_ms: float) -> str: + """Argus cycle name for a query entry, which also selects its results table. + + Queries of one workload that share an 'expected_p99_read_ms' land in the same table (and + validation rule) by construction -- there is no separate SCT-side grouping label. + """ + return f"{workload.name}_p99_{_format_ms(expected_p99_read_ms)}ms" + + +# Names for the values '_query_params' returns, in the same order, used when a row label has to be +# disambiguated by the query configuration. +_QUERY_PARAM_NAMES = ("limit", "concurrency", "rate") + + +def row_labels_for_step( + workload: SearchWorkload, queries: list, dataset_name: str, defaults: dict, record_count: int, step_number: int +) -> list[str]: + """Build the Argus row label for every query in a step, disambiguating collisions. + + Argus keys a result cell by (row, column), and ``add_result`` appends cells without + deduplicating while ``as_dict`` deduplicates ``rows_meta`` by name. Two entries that land in + the same table (see ``_cycle_name``) under the same label would therefore push conflicting + values into a single row -- their limit/concurrency/rate cannot keep them apart, since those + are reported as columns rather than folded into the label (see ``SEARCH_EXTRA_COLUMNS``). + + The label carries ``step #N``, the same 1-based ordinal the step's build row uses, for the same + reason that one does: record count alone does not identify a step. A step with an empty + ``shards`` list loads nothing, so it repeats its predecessor's count, and without the ordinal + its query rows would land on the predecessor's. It also lets a query row be lined up with the + build row it ran against. + + That leaves only collisions *within* a step, resolved in two cases: + + 1. A label that does not collide is returned unchanged. + 2. A colliding label is suffixed with the query configuration, e.g. + ``' | limit=5 concurrency=1 rate=50'``. Entries that collide *and* agree on every parameter + are genuinely indistinguishable, and those additionally get a ``' run #N'``. + + The suffix names every parameter rather than only the ones that differ within the collision + group: a differing-only suffix is shorter, but it depends on the whole group, so adding one + entry that varies a new parameter would rewrite the label of every other entry in the group -- + and Argus would lose their history. As written, the suffix depends on nothing but the entry, so + reordering never renames anything and adding an entry only affects rows sharing its label. Only + ``run #N`` is positional, and by then the entries are interchangeable by construction. + """ + labels = [ + f"{dataset_name} | {record_count:,} {workload.item_noun} | step #{step_number} | {query['set']}" + for query in queries + ] + params = [_query_params(query, defaults) for query in queries] + label_keys = [ + (_cycle_name(workload, _expected_p99_read_ms(query, defaults)), label) for query, label in zip(queries, labels) + ] + full_keys = [(*label_key, param) for label_key, param in zip(label_keys, params)] + + label_counts, full_counts = Counter(label_keys), Counter(full_keys) + seen_runs: Counter = Counter() + disambiguated = [] + for label, label_key, full_key, param in zip(labels, label_keys, full_keys, params): + if label_counts[label_key] < 2: + disambiguated.append(label) + continue + suffix = " ".join(f"{name}={value}" for name, value in zip(_QUERY_PARAM_NAMES, param)) + if full_counts[full_key] > 1: + seen_runs[full_key] += 1 + suffix = f"{suffix} run #{seen_runs[full_key]}" + disambiguated.append(f"{label} | {suffix}") + return disambiguated + + +def validate_plan_queries(datasets: list) -> None: + """Resolve every query entry of every dataset, so a bad plan fails before anything runs. + + Each of these raises on its own at the point the query is about to run -- but by then the step + has already loaded its shards and built its index, which on a real corpus is tens of minutes + spent to report a typo. Nothing here touches the cluster or the dataset files, so it is cheap + to do up front for the whole plan. + """ + for dataset in datasets: + defaults = dataset.get("defaults", {}) + for step_idx, step in enumerate(dataset.get("steps", [])): + for query in step.get("queries", []): + where = f"dataset {dataset.get('name')!r}, step #{step_idx + 1}" + if "set" not in query: + raise ValueError(f"Query entry in {where} has no 'set'") + try: + _checked_name(query["set"], "query set name") + _query_params(query, defaults) + _query_duration(query, defaults) + _expected_p99_read_ms(query, defaults) + except ValueError as exc: + raise ValueError(f"{exc} (in {where})") from exc + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + + +class SearchPerformanceTest(PerformanceRegressionTest): + """Multi-dataset, multi-step search benchmark driven by a YAML plan. + + Subclasses declare their index in ``WORKLOAD`` and expose a ``test_*`` method calling + ``run_search_benchmark``; everything below is the same for every search workload. + """ + + WORKLOAD: SearchWorkload = None + + def _run_latte(self, stress_cmd, files_to_stage=None, **kwargs): + """Run one latte command to completion and return its stress thread. + + Bypasses `run_stress_thread` so that the per-run data files of this command are staged into + the loader container. `files_to_stage` is a list of `(local_path, remote_path)` pairs. + """ + thread = self.run_latte_thread( + stress_cmd=stress_cmd, + extra_files_to_stage=files_to_stage or [], + # NOTE: every phase here is a single command -- one schema change, one shard, one index + # build -- so it belongs on one loader. Without this the thread fans out to *all* + # of them ('DockerBasedStressThread.configure_executer'), which on a multi-loader + # cluster would load every shard once per loader and report a record count, and + # the indexing throughput derived from it, off by that factor. + round_robin=True, + **kwargs, + ) + self.verify_stress_thread(thread) + return thread + + def _vector_store_node(self): + """Return the vector-store node this test talks to. + + Raises rather than returning None: every search test case sets 'n_vector_store_nodes: 1', so + a missing cluster is a misconfiguration, and failing here beats an AttributeError deep + inside an index build. + """ + vs_cluster = self.db_cluster.vector_store_cluster + if not vs_cluster or not vs_cluster.nodes: + raise RuntimeError("No vector-store node available; a search test requires 'n_vector_store_nodes' >= 1") + return vs_cluster.nodes[0] + + def _vector_store_api_client(self) -> VectorStoreClient: + return self._vector_store_node().get_vector_store_api_client() + + def _wait_for_vector_store_serving(self, timeout: float = DEFAULT_VECTOR_STORE_SERVING_WAIT): + """Block until the vector-store node reports node-level status SERVING. + + Cluster init already waits for the node to be ready (see + 'VectorStoreClusterMixin.wait_for_init'), but that accepts BOOTSTRAPPING too. Index timing + needs the node actually serving before it issues 'CREATE INDEX', so gate on that explicitly + rather than relying on the broader cluster-readiness check. + """ + vs_client = self._vector_store_api_client() + if not vs_client.wait_for_ready( + timeout=timeout, + check_interval=VECTOR_STORE_STATUS_POLL_INTERVAL_SECS, + required_statuses=("SERVING",), + ): + raise RuntimeError(f"Vector store did not reach SERVING within {timeout}s") + self.log.info("Vector store is SERVING") + + def _create_schema(self): + """Create keyspace and table for a dataset.""" + self.log.info("Creating schema") + self._run_latte(f"latte schema {self.WORKLOAD.script}", duration=_timeout_minutes(DEFAULT_SCHEMA_TIMEOUT)) + + def _load_shard(self, local_ds_dir, remote_ds_dir, shard_file, shard_count, max_load_wait): + """Load a single shard file into Scylla via the loader container.""" + params = self.WORKLOAD.params + self.log.info("Loading shard %s (%d %s)", shard_file, shard_count, self.WORKLOAD.item_noun) + self._run_latte( + # NOTE: latte's '-d' is a cycle count here, not a duration, so the phase gets an + # explicit 'duration' -- see DEFAULT_MAX_SHARD_LOAD. + stress_cmd=( + f"latte run -f load {self.WORKLOAD.script} " + f"-d {shard_count} " + rf"-P {params.dataset_dir}=\"{remote_ds_dir}\" " + rf"-P {params.records_file}=\"{shard_file}\" " + ), + files_to_stage=[ + (os.path.join(local_ds_dir, shard_file), os.path.join(remote_ds_dir, shard_file)), + ], + duration=_timeout_minutes(max_load_wait), + ) + + def _load_step_shards(self, step, local_ds_dir, remote_ds_dir, max_load_wait): + """Load all shard files for a step. Returns the record count.""" + workload = self.WORKLOAD + if "shards" in step: + shard_ids = _parse_shard_spec(step["shards"]) + shard_suffix = step.get("shard_suffix", workload.default_shard_suffix) + shard_files = ["shards/" + shard_suffix.format(sid) for sid in shard_ids] + else: + shard_files = [step.get(workload.step_records_file_key, workload.default_records_file)] + shard_files = [_checked_data_file(shard_file, "records file") for shard_file in shard_files] + + record_count = 0 + for shard_file in shard_files: + local_shard_path = os.path.join(local_ds_dir, shard_file) + shard_count = _count_tsv_lines(local_shard_path) + if shard_count == 0: + self.log.warning("Shard file %s is empty, skipping", local_shard_path) + continue + + self._load_shard(local_ds_dir, remote_ds_dir, shard_file, shard_count, max_load_wait) + record_count += shard_count + return record_count + + def _build_index(self, record_count, max_index_wait, index_name, keyspace) -> float | None: + """Build the index on the current table and return how long the build took, in seconds. + + latte still owns the DDL and decides when the index is usable (its 'build_index' probes the + index until it answers), but the reported duration does not come from latte: its clock is + whole seconds, and its probe loop is paced by retry backoff, so what it measures is "when a + retry after readiness happened to fire", several seconds late. Instead SCT reads + vector-store's own 'full scan' log lines afterwards -- see 'wait_for_index_build_seconds'. + + Returns None if those lines never turn up, which is reported as a missing measurement rather + than failing the build: the index is queryable either way, so the query phase can still run. + """ + params = self.WORKLOAD.params + self.log.info("Building index '%s' (%d %s)", index_name, record_count, self.WORKLOAD.item_noun) + with DbEventsFilter( + db_event=DatabaseLogEvent.DATABASE_ERROR, + line=r"vector_store_client.*(?:missing index|is not available yet)", + extra_time_to_expiration=120, + ): + self._run_latte( + stress_cmd=( + f"latte run -f build_index {self.WORKLOAD.script} " + f"-d 1 " + f'-P {params.index_name}=\\"{index_name}\\" ' + f"-P {params.record_count}={record_count} " + f"-P {params.max_index_wait}={max_index_wait} " + f"-P {params.min_probes}=3 " + ), + # The rune script gives up on its own after its own index wait; this is the outer + # bound for the case where it is the probe loop itself that stops making progress. + # The grace period keeps the two from expiring together: 'max_index_wait' is + # typically a whole number of minutes, so without it the script's own give-up and + # SCT's kill land in the same second and a clean "index never built" turns into a + # killed loader. + duration=_timeout_minutes(max_index_wait + INDEX_BUILD_TIMEOUT_GRACE_SECS), + ) + return wait_for_index_build_seconds(self._vector_store_node().system_log, keyspace, index_name) + + def _drop_index(self, index_name, keyspace, max_index_wait): + """Drop the current index via schema, and wait until vector-store confirms it is gone.""" + params = self.WORKLOAD.params + self.log.info("Dropping index '%s'", index_name) + self._run_latte( + f"latte schema {self.WORKLOAD.script} " + f"-P {params.drop_index}=true " + f'-P {params.index_name}=\\"{index_name}\\" ', + duration=_timeout_minutes(DEFAULT_SCHEMA_TIMEOUT), + ) + # NOTE: both names folded, since that is how vector-store knows them (see 'index_key'); + # querying with the unfolded name 404s forever, which would read as "already dropped". + self._vector_store_api_client().wait_for_index_absent( + keyspace.lower(), + index_name.lower(), + timeout=max_index_wait, + check_interval=VECTOR_STORE_STATUS_POLL_INTERVAL_SECS, + ) + + def _drop_table(self): + """Drop the current table via schema.""" + params = self.WORKLOAD.params + self.log.info("Dropping table") + self._run_latte( + f"latte schema {self.WORKLOAD.script} -P {params.schema_cleanup}=true ", + duration=_timeout_minutes(DEFAULT_SCHEMA_TIMEOUT), + ) + + def _run_search( + self, + local_ds_dir, + remote_ds_dir, + queries_file, + limit, + concurrency, + rate, + search_duration, + row_label, + expected_p99_read_ms, + query_example, + qrels_file=None, + ): + """Run a single search configuration with latency collection. + + Passing `qrels_file` turns on the relevance (accuracy) metrics of the rune script. + `row_label` is the Argus row; the table is selected by `expected_p99_read_ms` (see + `_cycle_name`), whose value also becomes the table's P99 validation rule and is stated in the + table description -- it is deliberately not a column, since it is the same for every row of + the table. `limit`/`concurrency`/`rate`/`query_example` do vary per row and are columns. + """ + workload = self.WORKLOAD + params = workload.params + cycle_name = _cycle_name(workload, expected_p99_read_ms) + # NOTE: this legend becomes the Argus table description (see 'send_result_to_argus') and is + # computed purely from the expected latency, so it is identical for every call sharing + # a cycle name -- unlike a per-query legend, it cannot make the description depend on + # whichever row happened to be submitted last. + table_description = ( + f"{workload.latency_legend} Expected P99 read <= {expected_p99_read_ms:g} ms. " + f"Query configuration (limit/concurrency/rate) and an example query are reported per row." + ) + error_thresholds = {"read": {"default": {"P99 read": {"fixed_limit": expected_p99_read_ms}}}} + extra_values = { + "limit": limit, + "concurrency": concurrency, + "rate": rate, + "query_example": query_example, + } + + @latency_calculator_decorator( + workload_type="read", + legend=table_description, + cycle_name=cycle_name, + row_name=row_label, + error_thresholds=error_thresholds, + extra_columns=SEARCH_EXTRA_COLUMNS, + ) + def _do_search(self): + files_to_stage = [ + (os.path.join(local_ds_dir, queries_file), os.path.join(remote_ds_dir, queries_file)), + ] + qrels_param = "" + if qrels_file: + files_to_stage.append((os.path.join(local_ds_dir, qrels_file), os.path.join(remote_ds_dir, qrels_file))) + qrels_param = rf"-P {params.qrels_file}=\"{qrels_file}\" " + rate_param = f"--rate={rate} " if rate else "" + self._run_latte( + # NOTE: '--duration' spelled out rather than '-d': 'get_timeout_from_stress_cmd' + # only recognises the long form, and without it the phase would fall back to + # the whole 'test_duration' as its timeout. + stress_cmd=( + f"latte run -f search {workload.script} " + f"--duration {search_duration} " + rf"-P {params.dataset_dir}=\"{remote_ds_dir}/\" " + rf"-P {params.queries_file}=\"{queries_file}\" " + f"{qrels_param}" + f"-P {params.compute_accuracy}={'true' if qrels_file else 'false'} " + f"-P {params.search_limit}={limit} " + f"{rate_param}--concurrency={concurrency} --retry-number 1 " + ), + files_to_stage=files_to_stage, + ) + # Read by 'latency_calculator_decorator': 'hdr_tags' selects the histograms to summarise, + # 'extra_values' fills the SEARCH_EXTRA_COLUMNS cells of this row. + return {"hdr_tags": [workload.hdr_tag], "extra_values": extra_values} + + with DbEventsFilter( + db_event=DatabaseLogEvent.DATABASE_ERROR, + line=r"failed to parse query", + extra_time_to_expiration=120, + ): + _do_search(self) + + def run_search_benchmark(self): + """Run every dataset of the plan 'search_test_config' points at.""" + self._wait_for_vector_store_serving() + + # NOTE: no fallback plan. Which datasets, shards and query sets to run is the whole + # definition of the test, and it is test-case specific, so there is nothing sensible + # to default to -- say so instead of failing later on a missing file. + config_name = self.params.get(TEST_CONFIG_PARAM) + if not config_name: + raise ValueError(f"'{TEST_CONFIG_PARAM}' is not set: the search test needs a plan to run") + config_path = resolve_test_config_path(config_name) + if not os.path.isfile(config_path): + raise FileNotFoundError(f"Test config '{config_name}' not found at {config_path}") + + with open(config_path, encoding="utf-8") as f: + config = yaml.safe_load(f) + self.log.info("Loaded search test config: %s", config_path) + + # An empty plan is a misconfiguration, not a zero-dataset run: it would load nothing, report + # nothing and still finish green, which is the one outcome nobody reads twice. + if not isinstance(config, dict): + raise ValueError(f"Test config '{config_name}' is not a YAML mapping") + datasets = config.get("datasets") + if not datasets: + raise ValueError(f"Test config '{config_name}' has no datasets to run") + # NOTE: the names have to be distinct. An index is named after its dataset and step, and its + # build time is read from the first matching 'full scan' pair in the log, so two + # datasets called the same thing would re-report the first one's build times. + names = [_checked_name(dataset["name"], "dataset name") for dataset in datasets] + if duplicates := sorted({name for name in names if names.count(name) > 1}): + raise ValueError(f"Duplicate dataset names in '{config_name}': {duplicates}") + validate_plan_queries(datasets) + + for dataset in datasets: + self._run_dataset(dataset) + + def _run_dataset(self, dataset): + """Load, index and query one dataset. + + Every dataset uses the keyspace/table named by 'latte_schema_parameters', dropped and + recreated here so each one starts from an empty table. + + Each step adds more shards on top of the previous ones and rebuilds the index, so that one + dataset yields index build and search results at several corpus sizes. An *empty* 'shards' + list loads nothing and only rebuilds on the corpus already there, e.g. to sample build-time + variance; an *absent* one falls back to the step's single-file key (an unsharded corpus -- + see 'local_smoke' in data_dir/latte/fts_search/local_config.yaml). + """ + workload = self.WORKLOAD + dataset_name = _checked_name(dataset["name"], "dataset name") + # A dataset is its steps: without one there is nothing to build and nothing to report, so + # say so here rather than dropping the table and finishing green. + steps = dataset.get("steps") + if not steps: + raise ValueError(f"Dataset '{dataset_name}' has no steps to run") + + local_ds_dir = _local_path(workload, dataset_name) + remote_ds_dir = f"{workload.remote_root}/{dataset_name}" + # The corpora are generated rather than tracked, so name the directory that is missing + # instead of failing further down on an unhelpful open() of a shard inside it. + if not os.path.isdir(local_ds_dir): + raise FileNotFoundError(f"Dataset '{dataset_name}' has no local directory at {local_ds_dir}") + + max_index_wait = dataset.get("max_index_wait_secs", DEFAULT_MAX_INDEX_WAIT) + max_load_wait = dataset.get("max_shard_load_secs", DEFAULT_MAX_SHARD_LOAD) + defaults = dataset.get("defaults", {}) + keyspace = (self.params.get("latte_schema_parameters") or {}).get("keyspace") or workload.default_keyspace + + self._drop_table() + self._create_schema() + + total_record_count = 0 + index_name = None + for step_idx, step in enumerate(steps): + if index_name is not None: + self._drop_index(index_name, keyspace, max_index_wait) + + total_record_count += self._load_step_shards(step, local_ds_dir, remote_ds_dir, max_load_wait) + + index_name = f"{workload.index_prefix}_{dataset_name}_{step_idx}" + build_seconds = self._build_index( + total_record_count, max_index_wait, index_name=index_name, keyspace=keyspace + ) + build_row_key = f"{dataset_name} | {total_record_count:,} {workload.item_noun} | build #{step_idx + 1}" + self._report_build_metrics(build_seconds, total_record_count, build_row_key) + + self._run_step_queries( + step, dataset_name, defaults, total_record_count, step_idx + 1, local_ds_dir, remote_ds_dir + ) + + if index_name is not None: + self._drop_index(index_name, keyspace, max_index_wait) + + def _run_step_queries(self, step, dataset_name, defaults, record_count, step_number, local_ds_dir, remote_ds_dir): + """Run every query set of a step against the index that was just built.""" + queries = step.get("queries", []) + # NOTE: row labels are resolved for the whole step up front so that repeated query configs + # can be told apart -- see row_labels_for_step(). + row_labels = row_labels_for_step(self.WORKLOAD, queries, dataset_name, defaults, record_count, step_number) + + for query, row_label in zip(queries, row_labels): + qset = _checked_name(query["set"], "query set name") + limit, concurrency, rate = _query_params(query, defaults) + expected_p99_read_ms = _expected_p99_read_ms(query, defaults) + queries_file = f"queries_{qset}.tsv" + qrels_file = f"qrels_{qset}.tsv" if query.get("qrels") else None + # A file the plan asks for but the corpus does not have would otherwise surface as a + # staging failure inside the loader container, phases into a run that has already loaded + # and indexed the corpus. Checked here instead -- after the dataset's download, so it + # covers an S3 corpus too -- to name the query set that asked for it. 'qrels: true' on a + # set that has no qrels file is the likely typo; the queries file is checked with it + # because a missing one is just as quiet ('_first_query_example' returns "" for it). + for needed in (queries_file, qrels_file): + if needed and not os.path.isfile(os.path.join(local_ds_dir, needed)): + raise ValueError( + f"Query set {qset!r} of step #{step_number} needs {needed!r}, which is not in {local_ds_dir}" + ) + + self._run_search( + local_ds_dir, + remote_ds_dir, + queries_file=queries_file, + limit=limit, + concurrency=concurrency, + rate=rate, + search_duration=_query_duration(query, defaults), + row_label=row_label, + expected_p99_read_ms=expected_p99_read_ms, + query_example=_first_query_example(local_ds_dir, queries_file), + qrels_file=qrels_file, + ) + + def _report_build_metrics(self, build_time: float | None, record_count, row_key): + """Send the index build time and indexing throughput of one step to Argus.""" + if build_time is None: + self.log.warning("No index build time measured; skipping build metrics for %s", row_key) + return + self.log.info("Index build time (vector-store full scan): %.4fs (%s)", build_time, row_key) + send_index_build_result( + argus_client=self.test_config.argus_client(), + result_table=self.WORKLOAD.build_result_table(), + count_column=self.WORKLOAD.build_count_column, + build_time=build_time, + count=record_count, + row_key=row_key, + ) diff --git a/test-cases/fts-search/fts-search-test-docker.yaml b/test-cases/fts-search/fts-search-test-docker.yaml new file mode 100644 index 00000000000..15435415c32 --- /dev/null +++ b/test-cases/fts-search/fts-search-test-docker.yaml @@ -0,0 +1,89 @@ +# Local correctness run of fts_test.FtsSearchTest.test_fts_search on the docker backend. +# +# Purpose: validate the *orchestration* of the FTS test (schema reset, shard +# staging, index build timing from vector-store's 'full scan' log lines, per-query-set search, +# Argus reporting with the expected_p99_read_ms table split) without provisioning AWS. +# The numbers this produces are meaningless -- 300 synthetic documents on a containerised Scylla. +# An aws test case for actual performance runs lands with the S3 dataset support. +# +# Prerequisites: +# 1. Build the vector-store image (needs 'fulltext_index' support): +# cd && docker build -t local/vector-store:fts . +# 2. Generate the tiny dataset (not tracked in git; a one-off after a fresh clone): +# python3 data_dir/latte/fts_search/generate_local_dataset.py +# +# Run: +# ./docker/env/hydra.sh run-test fts_test.FtsSearchTest.test_fts_search --backend docker \ +# --config test-cases/fts-search/fts-search-test-docker.yaml +# What to check afterwards is in docs/fts-search-test.md. +# +# Argus: outside Jenkins JOB_NAME is unset, so init_argus_client falls back to +# ReplayOnlyArgusSCTClient -- every submission is captured as JSONL in the run +# logdir and nothing is posted. `enable_argus` is deliberately left at its +# default of true so that latte's "schema once per script" behaviour matches a +# real run (see latte_thread.py:197). + +test_metadata: + description: >- + Local correctness run of the full-text search (BM25) performance test on the docker backend. + Loads a tiny synthetic dataset, builds a fulltext_index against a locally built vector-store + image, and runs latte query sets. Validates the test orchestration and Argus reporting rather + than producing meaningful performance numbers. + test_type: performance + tier: ondemand + duration_class: short + supported_backends: + - docker + stress_tools: + - latte + workload: read + nemesis_labels: [] + features: + - tablets + +test_duration: 180 + +# Required: latency_calculator_decorator returns early without it. +use_hdrhistogram: true + +n_db_nodes: 1 +n_loaders: 1 +# No monitoring stack locally: latency_calculator_decorator degrades gracefully without it, so the +# HDR results (from latte's .hdr files), the reactor stall counters (from the db logs) and the Argus +# rows are still collected -- only the Grafana screenshots and the Prometheus based metrics (the +# server side Scylla latencies) are absent. It also keeps the run from uploading the screenshots to +# the shared 's3://cloudius-jenkins-test' bucket. Set it to 1 when the dashboards are needed. +n_monitor_nodes: 0 +n_vector_store_nodes: 1 + +# Clean up after ourselves. 'execute_post_behavior' defaults to false because in Jenkins a separate +# pipeline stage destroys the resources -- there is no such stage behind a local run, so without this +# every run leaves its db and vector-store containers running (tester.py: "Resources will continue to +# run"). 'keep-on-failure' rather than 'destroy' so a failed run stays open for inspection. +execute_post_behavior: true +post_behavior_db_nodes: 'keep-on-failure' +post_behavior_loader_nodes: 'keep-on-failure' +post_behavior_monitor_nodes: 'keep-on-failure' +post_behavior_vector_store_nodes: 'keep-on-failure' + +# Scylla Manager is rejected outright on the docker backend +# (sct_config.py:_validate_docker_backend_parameters). +use_mgmt: false + +# On the docker backend `scylla_version` is used as the image tag. +docker_image: 'scylladb/scylla-nightly' +scylla_version: 'latest' + +# The docker backend takes a prebuilt image only -- it has no equivalent of the AWS source-build +# path, so build the fork locally and point at it here. Do NOT set vector_store_source_repo or +# vector_store_source_ref: they request a source build, which sct_config.py rejects on this backend. +vector_store_docker_image: 'local/vector-store' +vector_store_version: 'fts' + +user_prefix: 'fts-docker-local' + +search_test_config: 'data_dir/latte/fts_search/local_config.yaml' + +latte_schema_parameters: + keyspace: 'fts_bench' + table: 'documents' diff --git a/unit_tests/integration/test_search_perf_test.py b/unit_tests/integration/test_search_perf_test.py new file mode 100644 index 00000000000..5a4780b3b37 --- /dev/null +++ b/unit_tests/integration/test_search_perf_test.py @@ -0,0 +1,260 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Integration tests for the search performance flow, against real ScyllaDB and vector-store. + +Two levels, because they need different things to run: + + - dataset staging and the load, which need only a ScyllaDB container: the unit tests can assert that + 'LatteStressThread' calls 'send_files' for what it was given to stage, but whether the file arrives + where the rune script looks for it depends on the container's working directory, the '-P' names the + script declares and the path the flow interpolates, none of which a mock can check; + - the whole per-dataset cycle -- schema, load, index build, build time, index drop -- which needs a + vector-store that serves a full-text index, i.e. a nightly ScyllaDB and a vector-store built from + source, so it skips unless both images are already local. Override either with + SCT_FTS_IT_SCYLLA_IMAGE / SCT_FTS_IT_VECTOR_STORE_IMAGE. + +The second one runs the flow's own methods, not a re-implementation of them. Only two seams are +replaced: '_run_latte', which would otherwise need ClusterTester's loader provisioning, and +'_report_build_metrics', so the rows can be asserted instead of submitted to Argus. Everything between +them -- the step loop, the '-P' mapping, the index naming, reading the build time out of vector-store's +log, waiting for the drop -- is the code that ships. +""" + +import logging +import os +import subprocess + +import pytest + +import fts_test +import search_perf_test +from fts_test import FTS_WORKLOAD +from sdcm.stress.latte_thread import LatteStressThread +from unit_tests.lib.dummy_remote import LocalLoaderSetDummy + +pytestmark = [ + pytest.mark.usefixtures("events"), + pytest.mark.integration, +] + +LOGGER = logging.getLogger(__name__) + +# A full-text index needs a ScyllaDB that has one and a vector-store that serves it. Neither the +# released ScyllaDB nor the released vector-store does yet, and 'local/vector-store:fts' is built by +# hand (see docs/fts-search-test.md), so the cycle test skips rather than fails where either is +# absent -- both are checked, since a pull attempt during fixture setup is an error, not a skip. +FTS_SCYLLA_IMAGE = os.environ.get("SCT_FTS_IT_SCYLLA_IMAGE", "scylladb/scylla-nightly:latest") +FTS_VECTOR_STORE_IMAGE = os.environ.get("SCT_FTS_IT_VECTOR_STORE_IMAGE", "local/vector-store:fts") + +# Bodies distinctive enough that reading one back proves the row came from the staged file rather +# than from anything the script might have defaulted to. +DOCUMENTS = ( + ("doc_000001", "quasar obsidian zephyr staged from the test"), + ("doc_000002", "obsidian zephyr and nothing else"), + ("doc_000003", "zephyr alone"), +) + + +@pytest.fixture(name="staged_corpus") +def fixture_staged_corpus(tmp_path): + """A tiny documents TSV in the layout generate_local_dataset.py produces: '\\t'.""" + corpus = tmp_path / "documents_000.tsv" + corpus.write_text("".join(f"{doc_id}\t{body}\n" for doc_id, body in DOCUMENTS), encoding="utf-8") + return corpus + + +def test_a_staged_corpus_is_loaded_by_the_rune_script(request, docker_scylla, params, staged_corpus): + """The whole staging path end to end: the file reaches the container and latte loads it.""" + params["enable_argus"] = False + loader_set = LocalLoaderSetDummy(params=params) + + workload = FTS_WORKLOAD + remote_dir = f"{workload.remote_root}/integration" + remote_path = f"{remote_dir}/{staged_corpus.name}" + stress_cmd = ( + f"latte run -f load {workload.script} " + f"-d {len(DOCUMENTS)} " + rf"-P {workload.params.dataset_dir}=\"{remote_dir}\" " + rf"-P {workload.params.records_file}=\"{staged_corpus.name}\" " + ) + + latte_thread = LatteStressThread( + loader_set, + stress_cmd, + node_list=[docker_scylla], + timeout=5, + params=params, + extra_files_to_stage=[(str(staged_corpus), remote_path)], + ) + request.addfinalizer(latte_thread.kill) + + latte_thread.run() + latte_thread.get_results() + + keyspace, table = workload.default_keyspace, "documents" + with docker_scylla.parent_cluster.cql_connection_patient(docker_scylla) as session: + rows = {row.doc_id: row.body for row in session.execute(f"SELECT doc_id, body FROM {keyspace}.{table}")} + + assert rows == dict(DOCUMENTS), ( + "the rows in ScyllaDB must be exactly the staged file's -- a mismatch means the corpus latte " + "read was not the one this test staged" + ) + + +def test_an_unstaged_corpus_loads_nothing(request, docker_scylla, params, staged_corpus): + """The negative half, and the reason the thread subclass exists at all. + + Without staging, the file never reaches the container and the rune script's 'prepare' cannot read + it -- latte aborts. If this ever loads rows, the positive test above is no longer proving that the + corpus came from where it thinks. + + Asserted as "no rows", not as a raised exception: a failing latte command surfaces as an error + *event*, which 'ClusterTester.verify_stress_thread' turns into a test failure through + 'parse_results()'. 'get_results()' on its own returns normally, so a run that skipped + 'verify_stress_thread' would not notice. The flow always calls it -- see '_run_latte'. + """ + params["enable_argus"] = False + loader_set = LocalLoaderSetDummy(params=params) + + workload = FTS_WORKLOAD + stress_cmd = ( + f"latte run -f load {workload.script} " + f"-d {len(DOCUMENTS)} " + rf"-P {workload.params.dataset_dir}=\"{workload.remote_root}/absent\" " + rf"-P {workload.params.records_file}=\"{staged_corpus.name}\" " + ) + + latte_thread = LatteStressThread( + loader_set, + stress_cmd, + node_list=[docker_scylla], + timeout=5, + params=params, + extra_files_to_stage=[], + ) + request.addfinalizer(latte_thread.kill) + + latte_thread.run() + _, errors = latte_thread.parse_results() + assert errors, "a load whose corpus never reached the container must be reported as an error" + + keyspace, table = workload.default_keyspace, "documents" + with docker_scylla.parent_cluster.cql_connection_patient(docker_scylla) as session: + loaded = list(session.execute(f"SELECT doc_id FROM {keyspace}.{table}")) + assert not loaded, "nothing can have been loaded from a file that was never staged" + + +def _image_present(image: str) -> bool: + """Is the image already local? Checked at collection time, because the fixture that starts the + container would otherwise try to pull it during setup -- and a pull failure is an error, not a + skip.""" + try: + return subprocess.run(["docker", "image", "inspect", image], capture_output=True, check=False).returncode == 0 + except OSError: # no docker on this machine at all + return False + + +def _flow_over(params, docker_scylla, vs_cluster, loader_set, build_rows): + """The real FtsSearchTest with its two infrastructure seams replaced. + + Built with '__new__' rather than instantiated: ClusterTester's constructor is unittest's, and what + the phase methods actually use is a handful of attributes. + """ + flow = fts_test.FtsSearchTest.__new__(fts_test.FtsSearchTest) + flow.params = params + flow.log = LOGGER + flow.db_cluster = docker_scylla.parent_cluster + flow.db_cluster.vector_store_cluster = vs_cluster + + def run_latte(stress_cmd, files_to_stage=None, **_kwargs): + thread = LatteStressThread( + loader_set, + stress_cmd, + node_list=[docker_scylla], + timeout=10, + params=params, + extra_files_to_stage=files_to_stage or [], + ) + thread.run() + _, errors = thread.parse_results() + assert not errors, f"latte reported errors for {stress_cmd!r}: {errors}" + return thread + + flow._run_latte = run_latte + flow._report_build_metrics = lambda build_time, record_count, row_key: build_rows.append( + (row_key, build_time, record_count) + ) + return flow + + +@pytest.mark.skipif( + not (_image_present(FTS_VECTOR_STORE_IMAGE) and _image_present(FTS_SCYLLA_IMAGE)), + reason=( + f"{FTS_VECTOR_STORE_IMAGE} or {FTS_SCYLLA_IMAGE} is not available locally; " + "build or pull them per docs/fts-search-test.md" + ), +) +@pytest.mark.docker_scylla_args(scylla_docker_image=FTS_SCYLLA_IMAGE, vs_docker_image=FTS_VECTOR_STORE_IMAGE) +@pytest.mark.xdist_group("docker_heavy") +def test_a_dataset_is_loaded_indexed_and_reported(request, docker_scylla, docker_vector_store, params, tmp_path): + """The per-dataset cycle the whole test is built on, against a live vector-store. + + One dataset, two steps: load a shard and build an index over it, then load a second shard and + rebuild, so the cumulative record count and the drop-then-rebuild path are both exercised. What is + asserted is what a run reports -- a build row per step with a positive build time -- plus the rows + actually in ScyllaDB and the index being gone at the end. + """ + assert docker_vector_store, "the vector-store fixture did not start" + + params["enable_argus"] = False + dataset_name = "integration" + dataset_dir = tmp_path / dataset_name + (dataset_dir / "shards").mkdir(parents=True) + for shard, documents in enumerate((DOCUMENTS, DOCUMENTS[:2])): + (dataset_dir / "shards" / f"documents_{shard:03d}.tsv").write_text( + "".join(f"{doc_id}_{shard}\t{body}\n" for doc_id, body in documents), encoding="utf-8" + ) + # The flow resolves a dataset directory inside the repo; keep this run's data out of the tree. + request.getfixturevalue("monkeypatch").setattr( + search_perf_test, "_local_path", lambda _workload, *parts: str(tmp_path.joinpath(*parts)) + ) + + build_rows = [] + flow = _flow_over(params, docker_scylla, docker_vector_store, LocalLoaderSetDummy(params=params), build_rows) + flow._run_dataset( + { + "name": dataset_name, + "max_index_wait_secs": 300, + "steps": [{"shards": [0]}, {"shards": [1]}], + } + ) + + assert [row[0] for row in build_rows] == [ + f"{dataset_name} | {len(DOCUMENTS)} docs | build #1", + f"{dataset_name} | {len(DOCUMENTS) + 2} docs | build #2", + ], f"one build row per step, with cumulative counts: {build_rows}" + assert all(build_time > 0 for _, build_time, _ in build_rows), ( + f"every build time comes from vector-store's 'full scan' lines and must be positive: {build_rows}" + ) + + keyspace = FTS_WORKLOAD.default_keyspace + with flow.db_cluster.cql_connection_patient(docker_scylla) as session: + loaded = list(session.execute(f"SELECT doc_id FROM {keyspace}.documents")) + assert len(loaded) == len(DOCUMENTS) + 2, "both shards must be in the table" + + vs_client = docker_vector_store.nodes[0].get_vector_store_api_client() + last_index = f"{FTS_WORKLOAD.index_prefix}_{dataset_name}_1" + assert vs_client.get_index_status_or_none(keyspace, last_index.lower()) is None, ( + "the dataset loop drops the index it built last" + ) diff --git a/unit_tests/integration/test_vector_store.py b/unit_tests/integration/test_vector_store.py index ec9f3df8ddc..539f00571db 100644 --- a/unit_tests/integration/test_vector_store.py +++ b/unit_tests/integration/test_vector_store.py @@ -18,6 +18,8 @@ import logging import typing +from sdcm.utils.vector_store_index import index_key, wait_for_index_build_seconds + if typing.TYPE_CHECKING: from sdcm.utils.vector_store_client import VectorStoreClient from unit_tests.lib.dummy_remote import LocalScyllaClusterDummy @@ -183,3 +185,62 @@ def test_vector_search_error_handling(docker_scylla, docker_vector_store, params test_vector = [random.uniform(-1.0, 1.0) for _ in range(128)] with pytest.raises(Exception): vector_client.ann_search(keyspace="nonexistent", index="nonexistent_idx", vector=test_vector, limit=5) + + +def test_index_status_polling_spans_a_missing_index(docker_scylla, docker_vector_store, params): + """'get_index_status_or_none' and 'wait_for_index_absent' against a real vector-store. + + Both exist for the search performance tests, which build an index per step and drop it before the + next: vector-store discovers a fresh index asynchronously and forgets a dropped one just as + asynchronously, so a 404 has to read as "not there" rather than as an error. The unit tests mock + the HTTP layer; this checks the two states that are deterministic against a live service -- before + the index exists, and after it is dropped. + + Exercised through a vector index because that is what the released images support. Neither helper + knows or cares which kind of index it is; the full-text tests use the same two calls. + """ + db_cluster, vs_cluster = docker_scylla.parent_cluster, docker_vector_store + vector_client = vs_cluster.nodes[0].get_vector_store_api_client() + keyspace, index = "vector_test", "embeddings_vector_idx" + + # No such index yet: a 404 that must not raise. + assert vector_client.get_index_status_or_none(keyspace, index) is None + + create_vector_table(db_cluster) + insert_test_vectors(db_cluster, count=10) + wait_for_vector_indexing(vector_client) + + status = vector_client.get_index_status_or_none(keyspace, index) + assert status is not None and "status" in status, f"discovered index reported no status: {status}" + + with db_cluster.cql_connection_patient(db_cluster.nodes[0]) as session: + session.execute(f"DROP INDEX IF EXISTS {keyspace}.{index}") + + # Returns when vector-store has forgotten it; raises if it never does. + vector_client.wait_for_index_absent(keyspace, index, timeout=120) + assert vector_client.get_index_status_or_none(keyspace, index) is None + + +def test_index_build_time_is_read_from_the_vector_store_log(docker_scylla, docker_vector_store, params): + """'parse_full_scan_seconds' against the log a real vector-store writes. + + The search performance tests time an index build from vector-store's own "starting/finished full + scan" lines rather than from anything the stress tool prints. The unit tests parse captured lines; + what they cannot check is that the lines are still worded that way, still carry the case-folded + '.' key, and still reach the runner's copy of the log -- all of which this does. + """ + db_cluster, vs_cluster = docker_scylla.parent_cluster, docker_vector_store + vector_client = vs_cluster.nodes[0].get_vector_store_api_client() + keyspace, index = "vector_test", "embeddings_vector_idx" + + create_vector_table(db_cluster) + insert_test_vectors(db_cluster, count=10) + wait_for_vector_indexing(vector_client) + + build_seconds = wait_for_index_build_seconds(vs_cluster.nodes[0].system_log, keyspace, index, timeout=120) + assert build_seconds is not None, ( + "no 'starting'/'finished full scan' pair for " + f"{index_key(keyspace, index)} in {vs_cluster.nodes[0].system_log} -- either vector-store " + "reworded the lines or they are not reaching the runner" + ) + assert build_seconds >= 0.0 diff --git a/unit_tests/unit/test_argus_results.py b/unit_tests/unit/test_argus_results.py index ac1c6f8fbf6..dfae98c1bc6 100644 --- a/unit_tests/unit/test_argus_results.py +++ b/unit_tests/unit/test_argus_results.py @@ -15,7 +15,7 @@ from unittest.mock import MagicMock, call import pytest -from argus.client.generic_result import Cell, Status +from argus.client.generic_result import Cell, ColumnMetadata, ResultType, Status from sdcm.argus_results import ( ReactorStallStatsResult, @@ -95,3 +95,76 @@ def test_send_iotune_results_to_argus_skips_when_no_run(run): send_iotune_results_to_argus(argus_client=argus_mock, results={}, node=MagicMock(), params={}) argus_mock.submit_results.assert_not_called() + + +def test_extra_columns_and_extra_values_are_reported_once_per_row(): + """FTS-only usage: caller-supplied columns/values, without touching any existing table schema.""" + argus_mock = MagicMock() + result = { + "screenshots": [], + "duration_in_sec": 30, + "reactor_stalls_stats": {}, + "hdr_summary": { + "READ--fn--search": { + "percentile_90": 2.1, + "percentile_99": 4.5, + "throughput": 320, + } + }, + } + extra_columns = [ColumnMetadata(name="query_example", unit="", type=ResultType.TEXT)] + extra_values = {"query_example": "hello world"} + + send_result_to_argus( + argus_client=argus_mock, + workload="read", + name="fts_search_p99_10ms", + description="FTS BM25 full-text search query latency. Expected P99 read <= 10 ms.", + cycle="ds | 900 docs | term_common", + result=result, + start_time=1721564063.4528425, + extra_columns=extra_columns, + extra_values=extra_values, + ) + + (submitted_table,) = (c.args[0] for c in argus_mock.submit_results.call_args_list) + assert submitted_table.name == "read - fts_search_p99_10ms - latencies" + assert any(col.name == "query_example" for col in submitted_table.columns) + example_cells = [cell for cell in submitted_table.results if cell.column == "query_example"] + assert len(example_cells) == 1 + assert example_cells[0].value == "hello world" + assert example_cells[0].row == "ds | 900 docs | term_common" + + +def test_no_extra_columns_means_no_extra_columns_or_cells(): + """Existing callers (extra_columns/extra_values omitted) must see the unmodified table schema.""" + argus_mock = MagicMock() + result = { + "screenshots": [], + "duration_in_sec": 30, + "reactor_stalls_stats": {}, + "hdr_summary": {"READ--fn--search": {"percentile_90": 2.1, "percentile_99": 4.5, "throughput": 320}}, + } + + send_result_to_argus( + argus_client=argus_mock, + workload="read", + name="some_cycle", + description="", + cycle="row-1", + result=result, + ) + + # NOTE: against a literal, not against a fresh 'LatencyCalculatorReadResult()'. Had the extra + # columns leaked onto the shared class attribute, both sides would carry them and the + # comparison would pass while asserting nothing. + (submitted_table,) = (c.args[0] for c in argus_mock.submit_results.call_args_list) + assert [col.name for col in submitted_table.columns] == [ + "P90 read", + "P99 read", + "Throughput read", + "duration", + "start time", + "Overview", + "QA dashboard", + ] diff --git a/unit_tests/unit/test_fts_test.py b/unit_tests/unit/test_fts_test.py new file mode 100644 index 00000000000..c2d05cae8bb --- /dev/null +++ b/unit_tests/unit/test_fts_test.py @@ -0,0 +1,88 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Unit tests for the FTS half of the search performance flow: the workload descriptor. + +The flow itself is covered by test_search_perf_test.py. What is left to check here is that the +descriptor still matches the two things it names -- fts.rn and the Argus table -- because both can +drift out from under it. fts.rn is mirrored from scylladb/vector-store, so a rename there would +otherwise only surface as a latte run that ignores every '-P' it was given, hours into an AWS run. +""" + +import dataclasses +import re + +import pytest + +# NOTE: the test class is reached through the module rather than imported by name -- pytest's +# unittest collector picks up any 'unittest.TestCase' subclass in a test module's namespace, +# and 'FtsSearchTest' is one through ClusterTester. +import fts_test +from fts_test import FTS_BUILD_COUNT_COLUMN, FTS_WORKLOAD, FtsIndexBuildResult +from sdcm import sct_abs_path + +RUNE_PARAM_RE = re.compile(r"""latte::param!\(\s*["'](?P[^"']+)["']""") +RUNE_FUNCTION_RE = re.compile(r"^pub async fn (?P\w+)", re.MULTILINE) + + +@pytest.fixture(scope="module") +def script_source(): + with open(sct_abs_path(FTS_WORKLOAD.script), encoding="utf-8") as script: + return script.read() + + +def test_every_mapped_parameter_is_a_parameter_of_the_script(script_source): + """The descriptor's whole point is mapping onto fts.rn's own names, so check it does.""" + declared = set(RUNE_PARAM_RE.findall(script_source)) + mapped = {getattr(FTS_WORKLOAD.params, field.name) for field in dataclasses.fields(FTS_WORKLOAD.params)} + assert mapped <= declared, f"not parameters of {FTS_WORKLOAD.script}: {sorted(mapped - declared)}" + + +def test_the_hdr_tag_names_a_function_of_the_script(script_source): + """The tag is 'fn--'; latte emits nothing under it if that function is not there, + and the latency table would come out empty.""" + function = FTS_WORKLOAD.hdr_tag.removeprefix("fn--") + assert function in set(RUNE_FUNCTION_RE.findall(script_source)) + + +def test_the_phases_the_flow_invokes_exist(script_source): + """'load', 'build_index' and 'search' are the contract between the flow and any rune script.""" + functions = set(RUNE_FUNCTION_RE.findall(script_source)) + assert {"load", "build_index", "search"} <= functions + + +def test_the_step_record_file_key_shares_the_scripts_vocabulary(): + """A plan says 'documents_file:' because fts.rn calls it that; the two must not drift apart.""" + assert FTS_WORKLOAD.step_records_file_key == FTS_WORKLOAD.params.records_file + + +def test_index_build_table_counts_documents(): + """The count column is named once and reused, since Argus keys the table's history by it.""" + columns = FtsIndexBuildResult.Meta.Columns + assert FTS_BUILD_COUNT_COLUMN in {column.name for column in columns} + assert FTS_WORKLOAD.build_count_column == FTS_BUILD_COUNT_COLUMN + + +def test_argus_names_are_the_ones_the_history_is_under(): + """Renaming any of these silently starts a new, empty history in Argus.""" + assert FtsIndexBuildResult.Meta.name == "FTS Index Build Time" + assert FTS_WORKLOAD.name == "fts_search" # cycle names: 'fts_search_p99_10ms' + assert FTS_WORKLOAD.item_noun == "docs" # row labels: 'ds | 900 docs | term_common' + assert FTS_WORKLOAD.index_prefix == "fts_idx" + + +def test_the_test_case_entry_point_exists(): + """The name docs/fts-search-test.md tells you to run, and what the Jenkins job will name as its + sub_test once the aws test case lands. Renaming it breaks every documented invocation.""" + assert callable(fts_test.FtsSearchTest.test_fts_search) + assert fts_test.FtsSearchTest.WORKLOAD is FTS_WORKLOAD diff --git a/unit_tests/unit/test_latte_thread.py b/unit_tests/unit/test_latte_thread.py index 90c97f9e832..4b897be0e39 100644 --- a/unit_tests/unit/test_latte_thread.py +++ b/unit_tests/unit/test_latte_thread.py @@ -11,6 +11,8 @@ # # Copyright (c) 2021 ScyllaDB +import types + import pytest from sdcm import sct_abs_path @@ -100,6 +102,8 @@ def test_find_latte_fn_names(cmd, items): ("latte run /foo/bar.rn %smulti_get -q -r 500", "read"), ("latte run /foo/bar.rn %sdo_get_all -q -r 500", "read"), ("latte run /foo/bar.rn %sget_all,get_single -q -r 500", "read"), + ("latte run /foo/bar.rn %ssearch -q -r 500", "read"), + ("latte run /foo/bar.rn %sdo_search -q -r 500", "read"), ("latte run /foo/bar.rn %scounter_read -q -r 500", "counter_read"), ("latte run /foo/bar.rn %sread,write -q -r 500", "mixed"), ("latte run /foo/bar.rn %swrite:1,read:2 -q -r 500", "mixed"), @@ -139,3 +143,57 @@ def test_find_latte_tags(cmd, items): assert len(result) == len(items), f"Expected: {items}, Actual: {result}" for item in items: assert item in result + + +class _FakeRunner: + """Records what would be copied into the loader container. + + 'test -f' answers "already there" so the rune script directory is skipped and only the + per-invocation staging shows up; the 'latte schema' call answers like a successful run. + """ + + def __init__(self): + self.sent = [] + + def run(self, *_, **__): + return type("FakeResult", (), {"ok": True, "stdout": "", "stderr": ""}) + + def send_files(self, local_path, remote_path, **__): + self.sent.append((local_path, remote_path)) + + +def _staging_thread(extra_files_to_stage): + loader_set = types.SimpleNamespace( + get_db_auth=lambda: None, + test_config=types.SimpleNamespace(MULTI_REGION=False, tester_obj=lambda: object()), + ) + return LatteStressThread( + loader_set=loader_set, + stress_cmd="latte run -f load data_dir/latte/fts_search/fts.rn -d 100", + timeout=60, + node_list=["fake-db-node-1"], + params={"cluster_backend": "docker", "client_encrypt": False, "latte_schema_parameters": {}}, + extra_files_to_stage=extra_files_to_stage, + ) + + +def test_extra_files_are_staged_into_the_loader_container(): + """'build_stress_cmd' is the only point at which per-invocation data can reach the container: + the 'RemoteDocker' runner is created and destroyed per latte invocation, so anything staged + earlier is gone by the time the command runs.""" + runner = _FakeRunner() + + _staging_thread([("/local/documents_000.tsv", "/tmp/fts/ds/documents_000.tsv")]).build_stress_cmd( + runner, loader=None, hosts="10.0.0.1" + ) + + assert runner.sent == [("/local/documents_000.tsv", "/tmp/fts/ds/documents_000.tsv")] + + +def test_no_extra_files_stages_nothing(): + """The default has to stay empty: every existing caller passes no files at all.""" + runner = _FakeRunner() + + _staging_thread(None).build_stress_cmd(runner, loader=None, hosts="10.0.0.1") + + assert runner.sent == [] diff --git a/unit_tests/unit/test_search_perf_test.py b/unit_tests/unit/test_search_perf_test.py new file mode 100644 index 00000000000..5af28dfc81a --- /dev/null +++ b/unit_tests/unit/test_search_perf_test.py @@ -0,0 +1,627 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Unit tests for the pure helpers of the shared search performance flow. + +Exercised through a made-up workload rather than through the full-text one, so that a change in +what FTS happens to be called cannot make these pass or fail. fts_test's own descriptor is checked +in test_fts_test.py. +""" + +import logging +import os +import re + +import pytest + +import search_perf_test +from search_perf_test import ( + LatteScriptParams, + SearchWorkload, + _checked_data_file, + _checked_name, + _cycle_name, + _expected_p99_read_ms, + QUERY_EXAMPLE_MAX_CHARS, + _first_query_example, + _parse_shard_spec, + _query_params, + _timeout_minutes, + resolve_test_config_path, + row_labels_for_step, + validate_plan_queries, +) + +WORKLOAD = SearchWorkload( + name="search_bench", + base_dir="data_dir/latte/search_bench", + script="data_dir/latte/search_bench/bench.rn", + hdr_tag="fn--search", + item_noun="docs", + index_prefix="bench_idx", + default_keyspace="bench", + remote_root="/tmp/bench", + latency_legend="Search bench query latency.", + build_result_table=object, + build_count_column="record_count", + params=LatteScriptParams( + dataset_dir="data_dir", + records_file="records_file", + record_count="record_count", + queries_file="queries_file", + qrels_file="qrels_file", + search_limit="search_limit", + compute_accuracy="compute_accuracy", + index_name="index_name", + max_index_wait="max_index_wait_secs", + min_probes="min_successful_probes", + schema_cleanup="schema_cleanup", + drop_index="drop_index", + ), + step_records_file_key="records_file", + default_records_file="records.tsv", + default_shard_suffix="records_{:03d}.tsv", +) + +# --------------------------------------------------------------------------- +# Argus row-label disambiguation +# --------------------------------------------------------------------------- + +DEFAULTS = {"limit": 5, "concurrency": 32, "rate": 0, "expected_p99_read_ms": 10} + + +def _labels(queries, dataset_name="ds", defaults=None, record_count=900, step_number=1): + return row_labels_for_step( + WORKLOAD, queries, dataset_name, DEFAULTS if defaults is None else defaults, record_count, step_number + ) + + +def test_distinct_query_sets_keep_plain_labels(): + """Different query sets (or different record counts/datasets) must not be indexed -- + only limit/concurrency/rate no longer distinguish a row, since they are columns now.""" + queries = [ + {"set": "term_common", "concurrency": 1, "rate": 50}, + {"set": "term_medium"}, + {"set": "term_rare", "limit": 100}, + ] + assert _labels(queries) == [ + "ds | 900 docs | step #1 | term_common", + "ds | 900 docs | step #1 | term_medium", + "ds | 900 docs | step #1 | term_rare", + ] + + +def test_steps_with_an_equal_record_count_do_not_collide(): + """A step with an empty 'shards' list loads nothing, so it repeats its predecessor's record + count. Without the step ordinal its query rows would land on the predecessor's rows.""" + queries = [{"set": "term_common"}] + assert _labels(queries, record_count=300, step_number=1) != _labels(queries, record_count=300, step_number=2) + + +def test_repeated_query_set_is_disambiguated_by_its_config(): + """Entries for the same set/expected-latency group collide onto one row unless disambiguated. + + The ones that differ are told apart by their query configuration; only the two that are + identical in every parameter need a positional 'run #N'. + """ + queries = [ + {"set": "term_common", "concurrency": 1, "rate": 50}, + {"set": "term_common"}, + {"set": "term_medium"}, + {"set": "term_common"}, + ] + labels = _labels(queries) + + assert labels[0] == "ds | 900 docs | step #1 | term_common | limit=5 concurrency=1 rate=50" + assert labels[1] == "ds | 900 docs | step #1 | term_common | limit=5 concurrency=32 rate=0 run #1" + assert labels[2] == "ds | 900 docs | step #1 | term_medium" + assert labels[3] == "ds | 900 docs | step #1 | term_common | limit=5 concurrency=32 rate=0 run #2" + assert len(set(labels)) == len(labels) + + +def test_inserting_a_query_does_not_rename_the_other_rows(): + """The point of naming every parameter in the suffix rather than only those that differ. + + A differing-only suffix depends on the whole collision group, so an inserted entry that varies + a parameter the others agreed on would rename every one of their rows, and Argus would lose + their history. Here the inserted entry is the only one varying 'concurrency'. + """ + original = [ + {"set": "term_common", "rate": 50}, + {"set": "term_common", "rate": 10}, + ] + edited = [ + {"set": "term_common", "concurrency": 4}, # inserted at the head, varies a new parameter + *original, + ] + + before = _labels(original) + after = _labels(edited) + + assert after[1:] == before + assert after[0] == "ds | 900 docs | step #1 | term_common | limit=5 concurrency=4 rate=0" + + +def test_reordering_queries_does_not_rename_their_rows(): + """Reordering is pure permutation: no label depends on where its entry sits.""" + queries = [ + {"set": "term_common", "rate": 50}, + {"set": "term_common", "rate": 10}, + {"set": "term_common", "concurrency": 4}, + ] + assert sorted(_labels(queries)) == sorted(_labels(list(reversed(queries)))) + + +def test_same_label_in_different_tables_is_not_indexed(): + """Same set/record-count but a different 'expected_p99_read_ms' land in different tables, so + the shared row label is not a collision.""" + queries = [ + {"set": "natural", "expected_p99_read_ms": 50}, + {"set": "natural"}, # inherits DEFAULTS' expected_p99_read_ms of 10 -> a different table + ] + expected = "ds | 900 docs | step #1 | natural" + assert _labels(queries) == [expected, expected] + + +def test_three_way_collision_numbers_sequentially(): + queries = [{"set": "phrase", "expected_p99_read_ms": 50}] * 3 + assert _labels(queries, record_count=10) == [ + "ds | 10 docs | step #1 | phrase | limit=5 concurrency=32 rate=0 run #1", + "ds | 10 docs | step #1 | phrase | limit=5 concurrency=32 rate=0 run #2", + "ds | 10 docs | step #1 | phrase | limit=5 concurrency=32 rate=0 run #3", + ] + + +def test_record_count_is_thousands_separated(): + assert _labels([{"set": "natural"}], record_count=10_000_000)[0] == "ds | 10,000,000 docs | step #1 | natural" + + +def test_empty_step_returns_empty_list(): + assert _labels([], record_count=0) == [] + + +def test_missing_expected_latency_raises(): + """A query with no 'expected_p99_read_ms' on itself or the dataset defaults is a plan error, + not a silently-defaulted SCT value.""" + with pytest.raises(ValueError, match="expected_p99_read_ms"): + _labels([{"set": "natural"}], defaults={}, record_count=10) + + +# --------------------------------------------------------------------------- +# Resolving the expected latency and its Argus cycle/table name +# --------------------------------------------------------------------------- + + +def test_expected_p99_read_ms_query_overrides_defaults(): + assert _expected_p99_read_ms({"set": "s", "expected_p99_read_ms": 50}, {"expected_p99_read_ms": 10}) == 50.0 + + +def test_expected_p99_read_ms_falls_back_to_defaults(): + assert _expected_p99_read_ms({"set": "s"}, {"expected_p99_read_ms": 10}) == 10.0 + + +def test_expected_p99_read_ms_missing_raises(): + with pytest.raises(ValueError, match="term_common"): + _expected_p99_read_ms({"set": "term_common"}, {}) + + +@pytest.mark.parametrize( + "value", + (True, 0, -1, float("nan"), float("inf"), "fast", [10]), + ids=["bool", "zero", "negative", "nan", "inf", "string", "list"], +) +def test_expected_p99_read_ms_rejects_unusable_values(value): + """This value names the Argus table, becomes its P99 validation rule and is stated in its + description. 'float()' alone accepts all of these: a bool from an unquoted YAML 'yes', a rule no + run can satisfy, or a table named 'p99_infms'.""" + with pytest.raises(ValueError, match="expected a finite number > 0"): + _expected_p99_read_ms({"set": "term_common", "expected_p99_read_ms": value}, {}) + + +@pytest.mark.parametrize( + "value, expected_name", + ( + (10, "search_bench_p99_10ms"), + (50, "search_bench_p99_50ms"), + (12.5, "search_bench_p99_12_5ms"), + # '{:g}' would render this one as '1e+07'. + (10_000_000, "search_bench_p99_10000000ms"), + ), +) +def test_cycle_name_encodes_workload_and_expected_latency(value, expected_name): + assert _cycle_name(WORKLOAD, value) == expected_name + + +def test_cycle_name_groups_equal_expectations_together(): + """Two different queries sharing an expected value must land in the same table by construction.""" + assert _cycle_name(WORKLOAD, 10) == _cycle_name(WORKLOAD, 10.0) + + +# --------------------------------------------------------------------------- +# Validating a plan's query entries up front +# --------------------------------------------------------------------------- + + +def _plan(query, defaults=None): + return [{"name": "ds", "defaults": DEFAULTS if defaults is None else defaults, "steps": [{"queries": [query]}]}] + + +def test_validate_plan_queries_accepts_a_good_plan(): + validate_plan_queries(_plan({"set": "term_common", "limit": 100, "concurrency": 1, "rate": 50, "duration": "5m"})) + + +@pytest.mark.parametrize( + "query, message", + ( + ({"limit": 5}, "has no 'set'"), + ({"set": "term common"}, "query set name"), + ({"set": "t", "limit": 0}, "query limit"), + ({"set": "t", "limit": "5; rm -rf /"}, "query limit"), + ({"set": "t", "concurrency": 0}, "query concurrency"), + ({"set": "t", "rate": -1}, "query rate"), + ({"set": "t", "duration": "10"}, "query duration"), + ({"set": "t", "duration": "10s; id"}, "query duration"), + ), +) +def test_validate_plan_queries_rejects_a_bad_entry(query, message): + """Every one of these reaches a latte command line or an Argus name, so it is checked before the + run rather than quoted at each use.""" + with pytest.raises(ValueError, match=re.escape(message)): + validate_plan_queries(_plan(query)) + + +def test_validate_plan_queries_names_the_offending_step(): + with pytest.raises(ValueError, match=r"dataset 'ds', step #1"): + validate_plan_queries(_plan({"set": "t"}, defaults={})) + + +def test_validate_plan_queries_ignores_steps_without_queries(): + validate_plan_queries([{"name": "ds", "steps": [{"shards": []}, {"documents_file": "documents.tsv"}]}]) + + +def test_rate_zero_is_allowed_as_unthrottled(): + """0 means 'no --rate at all', not a missing value -- it is the DEFAULT_RATE.""" + assert _query_params({"set": "t", "rate": 0}, DEFAULTS)[2] == 0 + + +def test_booleans_are_not_accepted_as_numbers(): + """'True' is an int in Python and would reach the command line as 'concurrency=True'.""" + with pytest.raises(ValueError, match="query concurrency"): + _query_params({"set": "t", "concurrency": True}, DEFAULTS) + + +# --------------------------------------------------------------------------- +# Extracting a readable query example for the Argus 'query_example' column +# --------------------------------------------------------------------------- + + +def test_first_query_example_returns_the_text_column(tmp_path): + (tmp_path / "queries_term_common.tsv").write_text("q1\tfirst query text\nq2\tsecond\n", encoding="utf-8") + assert _first_query_example(str(tmp_path), "queries_term_common.tsv") == "first query text" + + +def test_first_query_example_skips_leading_blank_lines(tmp_path): + (tmp_path / "queries_natural.tsv").write_text("\n\nq1\tactual first\n", encoding="utf-8") + assert _first_query_example(str(tmp_path), "queries_natural.tsv") == "actual first" + + +def test_first_query_example_missing_file_returns_empty_string(tmp_path): + assert _first_query_example(str(tmp_path), "queries_missing.tsv") == "" + + +def test_first_query_example_without_a_tab_returns_whole_line(tmp_path): + (tmp_path / "queries_odd.tsv").write_text("just one column\n", encoding="utf-8") + assert _first_query_example(str(tmp_path), "queries_odd.tsv") == "just one column" + + +def test_first_query_example_truncates_a_long_query(tmp_path): + """The Argus cell is bounded here, not by the corpus the plan happens to name.""" + (tmp_path / "queries_long.tsv").write_text(f"q1\t{'a' * 900}\n", encoding="utf-8") + + example = _first_query_example(str(tmp_path), "queries_long.tsv") + + assert len(example) == QUERY_EXAMPLE_MAX_CHARS + assert example.endswith("...") + + +def test_first_query_example_keeps_a_query_at_the_cap_intact(tmp_path): + (tmp_path / "queries_exact.tsv").write_text(f"q1\t{'a' * QUERY_EXAMPLE_MAX_CHARS}\n", encoding="utf-8") + + example = _first_query_example(str(tmp_path), "queries_exact.tsv") + + assert example == "a" * QUERY_EXAMPLE_MAX_CHARS + + +# --------------------------------------------------------------------------- +# Waiting for the vector-store node to actually serve +# --------------------------------------------------------------------------- + + +class _FakeTester: + """A minimal stand-in exposing just what the polling methods use, called via the unbound + 'SearchPerformanceTest' methods below -- not a subclass, so pytest's unittest collector (which + matches any 'unittest.TestCase' subclass regardless of name) does not pick it up as a test. + """ + + def __init__(self, ready): + self.log = __import__("logging").getLogger("test") + self._ready = ready + self.asked = {} + + def _vector_store_api_client(self): + tester = self + + class _Client: + @staticmethod + def wait_for_ready(**kwargs): + tester.asked = kwargs + return tester._ready + + return _Client() + + +def test_wait_for_vector_store_serving_requires_serving_only(): + tester = _FakeTester(ready=True) + search_perf_test.SearchPerformanceTest._wait_for_vector_store_serving(tester, timeout=10) + assert tester.asked["required_statuses"] == ("SERVING",) + + +def test_wait_for_vector_store_serving_raises_when_it_never_serves(): + tester = _FakeTester(ready=False) + with pytest.raises(RuntimeError, match="did not reach SERVING"): + search_perf_test.SearchPerformanceTest._wait_for_vector_store_serving(tester, timeout=10) + + +# --------------------------------------------------------------------------- +# Resolving the plan param to a local file +# --------------------------------------------------------------------------- + + +def test_a_relative_path_resolves_from_the_sct_root(): + """The plan is named the way every other file a test case points at is -- repo-relative -- so + a plan outside the workload's own data directory needs no new param.""" + resolved = resolve_test_config_path("data_dir/latte/search_bench/local_config.yaml") + assert resolved.endswith(os.path.join("data_dir", "latte", "search_bench", "local_config.yaml")) + assert os.path.isabs(resolved) + + +def test_absolute_path_is_used_as_is(tmp_path): + plan = tmp_path / "my_plan.yaml" + plan.write_text("datasets: []\n", encoding="utf-8") + assert resolve_test_config_path(str(plan)) == str(plan) + + +# --------------------------------------------------------------------------- +# Shard specs +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "spec, expected", + ( + ([], []), + ([0], [0]), + ([0, 1, 2], [0, 1, 2]), + (["0..2"], [0, 1, 2]), + (["0..1", 5, "8..9"], [0, 1, 5, 8, 9]), + (["3..3"], [3]), + ), +) +def test_shard_spec_normalizes_ints_and_ranges(spec, expected): + assert _parse_shard_spec(spec) == expected + + +@pytest.mark.parametrize("spec", (["0-2"], ["0..a"], ["..2"], ["3..1"], [1.5], [None], [{"from": 0}], [True])) +def test_shard_spec_rejects_unrecognised_entries(spec): + """Dropping these silently would load a smaller corpus than the plan asked for, and the run + would then report plausible numbers against the wrong record count.""" + with pytest.raises(ValueError): + _parse_shard_spec(spec) + + +@pytest.mark.parametrize( + "spec", + ([1, 1], ["0..2", 2], ["0..2", "2..4"], [3, "0..5"]), + ids=["repeated-int", "int-inside-a-range", "overlapping-ranges", "int-covered-by-a-later-range"], +) +def test_shard_spec_rejects_duplicates(spec): + """Loading a shard twice inflates the record count, and the throughput derived from it, while + the table itself gains nothing -- the same silent misreport as an under-load, in the other + direction.""" + with pytest.raises(ValueError, match="appear more than once"): + _parse_shard_spec(spec) + + +# --------------------------------------------------------------------------- +# Validation of plan supplied names +# +# The plan may come from an arbitrary S3 URL, and its names reach a CQL index name, a shell command +# and a local path. Validate them on the way in rather than quoting for three contexts at once. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", ("10M_20tok", "local_tiny", "term_common", "natural")) +def test_plain_names_are_accepted(name): + assert _checked_name(name, "dataset name") == name + + +@pytest.mark.parametrize( + "name", + ( + 'evil"; rm -rf /; echo "', + "with space", + "with-dash", + "with.dot", + "../escape", + "", + None, + ), +) +def test_unsafe_names_are_rejected(name): + with pytest.raises(ValueError, match="Invalid dataset name"): + _checked_name(name, "dataset name") + + +@pytest.mark.parametrize("name", ("documents.tsv", "shards/documents_000.tsv", "a-b_c.1.tsv")) +def test_plain_data_file_names_are_accepted(name): + assert _checked_data_file(name, "records file") == name + + +@pytest.mark.parametrize( + "name", + ( + "../../../etc/passwd", + "/etc/passwd", + "shards/../../escape.tsv", + "$(id).tsv", + "with space.tsv", + "", + ), +) +def test_unsafe_data_file_names_are_rejected(name): + with pytest.raises(ValueError, match="Invalid records file"): + _checked_data_file(name, "records file") + + +# --------------------------------------------------------------------------- +# Phase timeouts +# +# Without an explicit duration 'run_latte_thread' falls back to the whole 'test_duration', so the +# phases carrying no '--duration' pass one of their own. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "seconds, expected", + ((0, 1), (1, 1), (60, 1), (61, 2), (600, 10), (1800, 30), (3600, 60)), +) +def test_timeout_minutes_rounds_up_and_never_returns_zero(seconds, expected): + assert _timeout_minutes(seconds) == expected + + +# --------------------------------------------------------------------------- +# Every phase runs on a single loader +# +# 'run_latte_thread' fans a thread out to every loader unless it is asked not to, and each of these +# phases is one command -- so a multi-loader cluster would run all of them once per loader. +# --------------------------------------------------------------------------- + + +class _RecordingTester: + """Records what '_run_latte' asks 'run_latte_thread' for.""" + + def __init__(self): + self.asked = {} + + def run_latte_thread(self, **kwargs): + self.asked = kwargs + return "thread" + + def verify_stress_thread(self, thread): + assert thread == "thread" + + +def test_every_phase_runs_on_one_loader(): + tester = _RecordingTester() + search_perf_test.SearchPerformanceTest._run_latte(tester, "latte schema bench.rn", duration=1) + assert tester.asked["round_robin"] is True + + +# --------------------------------------------------------------------------- +# Plan validation +# --------------------------------------------------------------------------- + + +class _PlanTester: + """Enough of the flow to load a plan: the datasets it would run are recorded, not run.""" + + WORKLOAD = WORKLOAD + + def __init__(self, plan_path): + self.log = logging.getLogger("test") + self.params = {search_perf_test.TEST_CONFIG_PARAM: str(plan_path)} + self.ran = [] + + def _wait_for_vector_store_serving(self): + pass + + def _run_dataset(self, dataset): + self.ran.append(dataset["name"]) + + +def _run_plan(tmp_path, plan): + plan_path = tmp_path / "plan.yaml" + plan_path.write_text(plan, encoding="utf-8") + tester = _PlanTester(plan_path) + search_perf_test.SearchPerformanceTest.run_search_benchmark(tester) + return tester + + +def test_a_plan_runs_its_datasets_in_order(tmp_path): + tester = _run_plan(tmp_path, "datasets:\n - name: first\n - name: second\n") + assert tester.ran == ["first", "second"] + + +def test_repeated_dataset_names_are_rejected(tmp_path): + """An index is named after its dataset and step, and its build time is read from the first + matching 'full scan' pair in the log -- so a repeated name would silently re-report the first + dataset's build times against the second one's document counts.""" + with pytest.raises(ValueError, match="Duplicate dataset names.*same"): + _run_plan(tmp_path, "datasets:\n - name: same\n - name: other\n - name: same\n") + + +@pytest.mark.parametrize( + "plan, expected", + ( + ("datasets: []\n", "no datasets to run"), + ("{}\n", "no datasets to run"), + ("# only a comment\n", "not a YAML mapping"), + ("- name: first\n", "not a YAML mapping"), + ), +) +def test_a_plan_that_would_run_nothing_is_rejected(tmp_path, plan, expected): + """Such a run loads nothing, reports nothing and still finishes green, which reads as a passing + benchmark rather than as the misconfiguration it is.""" + with pytest.raises(ValueError, match=expected): + _run_plan(tmp_path, plan) + + +def test_an_unset_plan_is_rejected(tmp_path): + tester = _PlanTester(tmp_path / "plan.yaml") + tester.params = {search_perf_test.TEST_CONFIG_PARAM: ""} + with pytest.raises(ValueError, match="is not set"): + search_perf_test.SearchPerformanceTest.run_search_benchmark(tester) + + +def test_a_missing_plan_file_is_rejected(tmp_path): + tester = _PlanTester(tmp_path / "absent.yaml") + with pytest.raises(FileNotFoundError, match="not found at"): + search_perf_test.SearchPerformanceTest.run_search_benchmark(tester) + + +def test_a_dataset_with_no_local_corpus_says_which_directory_is_missing(): + """The corpora are generated, not tracked, so a fresh clone that skipped the generator has to + hear which directory to produce rather than an open() failure on a shard inside it.""" + tester = _PlanTester("unused") + with pytest.raises(FileNotFoundError, match="no local directory"): + search_perf_test.SearchPerformanceTest._run_dataset( + tester, {"name": "never_generated", "steps": [{"shards": [0]}]} + ) + + +def test_a_dataset_with_no_steps_is_rejected(): + """It would drop and recreate the table, build nothing and report nothing, so the run would + pass while measuring an index it never built.""" + tester = _PlanTester("unused") + with pytest.raises(ValueError, match="no steps to run"): + search_perf_test.SearchPerformanceTest._run_dataset(tester, {"name": "local_tiny", "steps": []}) diff --git a/unit_tests/unit/test_tester.py b/unit_tests/unit/test_tester.py index 083c1cf7d5a..92f92917782 100644 --- a/unit_tests/unit/test_tester.py +++ b/unit_tests/unit/test_tester.py @@ -25,6 +25,7 @@ from sdcm.sct_events.base import SctEvent from sdcm.sct_events.health import ClusterHealthValidatorEvent from sdcm.sct_events.system import TestFrameworkEvent +from sdcm.stress.latte_thread import LatteStressThread from sdcm.test_config import TestConfig from sdcm.tester import ClusterTester, silence from sdcm.utils.common import get_post_behavior_actions @@ -807,3 +808,54 @@ def test_get_cluster_docker_nonzero_monitor_nodes_builds_monitor_set_docker(tmp_ cluster_docker_mock.MonitorSetDocker.assert_called_once() assert tester.monitors is cluster_docker_mock.MonitorSetDocker.return_value + + +# run_latte_thread: a test may swap in a 'LatteStressThread' subclass + + +def _fake_latte_tester(): + """The 'ClusterTester' attributes 'run_latte_thread' reads, and nothing else.""" + return types.SimpleNamespace( + loaders="loader-set", + db_cluster=types.SimpleNamespace(nodes=["node-1"]), + _stress_duration=None, + params={"stop_test_on_stress_failure": True}, + get_duration=lambda duration: 60 * duration, + ) + + +def test_run_latte_thread_forwards_the_files_to_stage(): + """A caller with per-invocation data -- a dataset shard -- reaches the thread through here, so + it does not have to reimplement this helper's timeout resolution and loader fan-out.""" + files = [("/local/documents_000.tsv", "/tmp/fts/ds/documents_000.tsv")] + with ( + patch.object(LatteStressThread, "__init__", return_value=None) as mock_init, + patch.object(LatteStressThread, "run", return_value="thread-pool") as mock_run, + ): + result = ClusterTester.run_latte_thread( + _fake_latte_tester(), + stress_cmd="latte run -f load data_dir/latte/x.rn", + duration=10, + extra_files_to_stage=files, + ) + + assert result == "thread-pool" + assert mock_run.call_count == 1 + assert mock_init.call_args.kwargs["extra_files_to_stage"] == files + # the resolved timeout is the reason to go through the helper instead of constructing directly + assert mock_init.call_args.kwargs["timeout"] == 600 + + +def test_run_latte_thread_stages_nothing_by_default(): + with ( + patch.object(LatteStressThread, "__init__", return_value=None) as mock_init, + patch.object(LatteStressThread, "run", return_value="thread-pool") as mock_run, + ): + result = ClusterTester.run_latte_thread( + _fake_latte_tester(), stress_cmd="latte run -f search data_dir/latte/x.rn", duration=10 + ) + + assert result == "thread-pool" + assert mock_run.call_count == 1 + assert mock_init.call_args.kwargs["extra_files_to_stage"] is None + assert mock_init.call_args.kwargs["timeout"] == 600 diff --git a/unit_tests/unit/test_utils_decorators.py b/unit_tests/unit/test_utils_decorators.py index 4cba7e5b28c..a8510870536 100644 --- a/unit_tests/unit/test_utils_decorators.py +++ b/unit_tests/unit/test_utils_decorators.py @@ -4,6 +4,7 @@ import pytest from google.api_core.exceptions import ServiceUnavailable +from argus.client.generic_result import ColumnMetadata, ResultType from sdcm.exceptions import UnsupportedNemesis from sdcm.provision.provisioner import ProvisionUnrecoverableError from sdcm.sct_events import Severity @@ -234,3 +235,88 @@ def test_latency_calculator_decorator_with_monitoring_set(tmp_path): assert cycle["Scylla P99_read - node-1"] == 1.5 assert len(cycle["screenshots"]) == 1 assert tester.monitors.screenshot_requests + + +# error_thresholds override / extra_columns / extra_values -- used by fts_test.py so that a plan- +# supplied expected latency becomes the table's validation rule and per-row metadata columns, +# without requiring every caller to route through 'latency_decorator_error_thresholds'. + + +def test_error_thresholds_override_bypasses_test_params(tmp_path): + tester = FakeLatencyTester(latency_results_file=tmp_path / "latency_results.json", monitors=FakeMonitorSet()) + tester.params["latency_decorator_error_thresholds"] = {"read": {"default": {"P99 read": {"fixed_limit": 999}}}} + override = {"read": {"default": {"P99 read": {"fixed_limit": 50}}}} + + @latency_calculator_decorator( + workload_type="read", legend="fts search", cycle_name="fts_search", row_name="row-1", error_thresholds=override + ) + def _do_search(_self): + return {"hdr_tags": ["fn--search"]} + + with ( + patch("sdcm.tester.ClusterTester", FakeLatencyTester), + patch("sdcm.utils.decorators.EventCounterContextManager", FakeEventCounter), + patch("sdcm.utils.latency.collect_latency"), + patch("sdcm.utils.decorators.send_result_to_argus") as send_to_argus_mock, + ): + _do_search(tester) + + assert send_to_argus_mock.call_args.kwargs["error_thresholds"] == override + + +def test_error_thresholds_defaults_to_test_params_when_not_overridden(tmp_path): + tester = FakeLatencyTester(latency_results_file=tmp_path / "latency_results.json", monitors=FakeMonitorSet()) + configured = {"read": {"default": {"P99 read": {"fixed_limit": 10}}}} + tester.params["latency_decorator_error_thresholds"] = configured + + with ( + patch("sdcm.tester.ClusterTester", FakeLatencyTester), + patch("sdcm.utils.decorators.EventCounterContextManager", FakeEventCounter), + patch("sdcm.utils.latency.collect_latency"), + patch("sdcm.utils.decorators.send_result_to_argus") as send_to_argus_mock, + ): + _run_decorated_search(tester) + + assert send_to_argus_mock.call_args.kwargs["error_thresholds"] == configured + + +def test_extra_columns_and_extra_values_are_forwarded(tmp_path): + tester = FakeLatencyTester(latency_results_file=tmp_path / "latency_results.json", monitors=FakeMonitorSet()) + extra_columns = [ColumnMetadata(name="query_example", unit="", type=ResultType.TEXT)] + + @latency_calculator_decorator( + workload_type="read", + legend="fts search", + cycle_name="fts_search", + row_name="row-1", + extra_columns=extra_columns, + ) + def _do_search(_self): + return {"hdr_tags": ["fn--search"], "extra_values": {"query_example": "hello world"}} + + with ( + patch("sdcm.tester.ClusterTester", FakeLatencyTester), + patch("sdcm.utils.decorators.EventCounterContextManager", FakeEventCounter), + patch("sdcm.utils.latency.collect_latency"), + patch("sdcm.utils.decorators.send_result_to_argus") as send_to_argus_mock, + ): + _do_search(tester) + + assert send_to_argus_mock.call_args.kwargs["extra_columns"] == extra_columns + assert send_to_argus_mock.call_args.kwargs["extra_values"] == {"query_example": "hello world"} + + +def test_extra_columns_defaults_to_none_for_existing_callers(tmp_path): + """Callers that do not pass 'extra_columns' must not change the call to 'send_result_to_argus'.""" + tester = FakeLatencyTester(latency_results_file=tmp_path / "latency_results.json", monitors=FakeMonitorSet()) + + with ( + patch("sdcm.tester.ClusterTester", FakeLatencyTester), + patch("sdcm.utils.decorators.EventCounterContextManager", FakeEventCounter), + patch("sdcm.utils.latency.collect_latency"), + patch("sdcm.utils.decorators.send_result_to_argus") as send_to_argus_mock, + ): + _run_decorated_search(tester) + + assert send_to_argus_mock.call_args.kwargs["extra_columns"] is None + assert send_to_argus_mock.call_args.kwargs["extra_values"] is None diff --git a/unit_tests/unit/test_vector_store_client.py b/unit_tests/unit/test_vector_store_client.py new file mode 100644 index 00000000000..8423d2ecccd --- /dev/null +++ b/unit_tests/unit/test_vector_store_client.py @@ -0,0 +1,108 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Unit tests for the index/readiness polling of VectorStoreClient. + +Used by the search performance tests to poll through the discovery gap between 'CREATE INDEX' and +the index becoming visible to vector-store (404 until then), and the same gap on the way out -- see +docs/fts-search-test.md. +""" + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from sdcm.utils import vector_store_client as vector_store_client_module +from sdcm.utils.vector_store_client import VectorStoreClient + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + monkeypatch.setattr(vector_store_client_module.time, "sleep", lambda _seconds: None) + + +def _http_error(status_code): + response = MagicMock() + response.status_code = status_code + error = requests.exceptions.HTTPError(f"{status_code} error") + error.response = response + return error + + +def test_get_index_status_or_none_returns_status_on_success(): + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_index_status", return_value={"status": "SERVING", "count": 5}) as mocked: + assert client.get_index_status_or_none("ks", "idx") == {"status": "SERVING", "count": 5} + mocked.assert_called_once_with("ks", "idx") + + +def test_get_index_status_or_none_returns_none_on_404(): + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_index_status", side_effect=_http_error(404)): + assert client.get_index_status_or_none("ks", "idx") is None + + +def test_get_index_status_or_none_reraises_non_404_http_errors(): + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_index_status", side_effect=_http_error(500)): + with pytest.raises(requests.exceptions.HTTPError): + client.get_index_status_or_none("ks", "idx") + + +def test_get_index_status_or_none_reraises_when_response_is_missing(): + """An HTTPError with no attached response (e.g. a connection-level failure) must not be + mistaken for 'not discovered yet'.""" + client = VectorStoreClient(base_url="http://vs.example") + error = requests.exceptions.HTTPError("no response") + error.response = None + with patch.object(client, "get_index_status", side_effect=error): + with pytest.raises(requests.exceptions.HTTPError): + client.get_index_status_or_none("ks", "idx") + + +def test_wait_for_index_absent_returns_once_the_index_is_gone(): + client = VectorStoreClient(base_url="http://vs.example") + statuses = [{"status": "SERVING", "count": 5}, None] + with patch.object(client, "get_index_status_or_none", side_effect=statuses): + client.wait_for_index_absent("ks", "idx", timeout=10) + + +def test_wait_for_index_absent_raises_past_the_deadline(): + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_index_status_or_none", return_value={"status": "SERVING"}): + with pytest.raises(RuntimeError, match="was not dropped"): + client.wait_for_index_absent("ks", "idx", timeout=0) + + +def test_wait_for_index_absent_treats_a_failed_request_as_still_there(): + """A network hiccup must not be mistaken for the index having disappeared.""" + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_index_status_or_none", side_effect=RuntimeError("connection reset")): + with pytest.raises(RuntimeError, match="request failed: connection reset"): + client.wait_for_index_absent("ks", "idx", timeout=0) + + +def test_wait_for_ready_accepts_bootstrapping_by_default(): + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_status", return_value="BOOTSTRAPPING"): + assert client.wait_for_ready(timeout=1, check_interval=0) is True + + +def test_wait_for_ready_can_require_serving_only(): + """What an index-timing caller needs: 'answers and is catching up' is not good enough.""" + client = VectorStoreClient(base_url="http://vs.example") + with patch.object(client, "get_status", return_value="BOOTSTRAPPING"): + assert client.wait_for_ready(timeout=0.01, check_interval=0, required_statuses=("SERVING",)) is False + with patch.object(client, "get_status", return_value="SERVING"): + assert client.wait_for_ready(timeout=1, check_interval=0, required_statuses=("SERVING",)) is True diff --git a/unit_tests/unit/test_vector_store_index.py b/unit_tests/unit/test_vector_store_index.py new file mode 100644 index 00000000000..36cb2880753 --- /dev/null +++ b/unit_tests/unit/test_vector_store_index.py @@ -0,0 +1,168 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright (c) 2026 ScyllaDB + +"""Unit tests for reading a vector-store index build time out of the node's log.""" + +import pytest + +from sdcm.utils import vector_store_index +from sdcm.utils.vector_store_index import ( + index_build_columns, + index_key, + parse_full_scan_seconds, + send_index_build_result, + wait_for_index_build_seconds, +) + +# Verbatim lines from real runs, so the parser is tested against both formats 'BaseNode.system_log' +# can resolve to. The docker one is the raw tracing line; the aws one carries a log-shipper prefix +# whose timestamp has no 'Z' -- which is what the regex uses to pick the tracing timestamp. +AWS_SCAN_LINES = ( + "2026-07-30T23:05:37.908 fts-search-vs-node-1 !INFO | vector-store[12767] " + "2026-07-30T23:05:37.908018Z INFO db:db-process:db_index{fts_bench.fts_idx_10m_20tok_0}: " + "starting full scan on fts_bench.fts_idx_10m_20tok_0\n" + "2026-07-30T23:06:43.914 fts-search-vs-node-1 !INFO | vector-store[12767] " + "2026-07-30T23:06:43.914698Z INFO db:db-process:db_index{fts_bench.fts_idx_10m_20tok_0}: " + "finished full scan on fts_bench.fts_idx_10m_20tok_0\n" +) +DOCKER_SCAN_LINES = ( + "2026-07-30T21:28:00.727916Z INFO db:db-process:db_index{fts_bench.fts_idx_local_tiny_0}: " + "starting full scan on fts_bench.fts_idx_local_tiny_0\n" + "2026-07-30T21:28:03.289113Z INFO db:db-process:db_index{fts_bench.fts_idx_local_tiny_0}: " + "finished full scan on fts_bench.fts_idx_local_tiny_0\n" +) + + +def _write_log(tmp_path, content): + path = tmp_path / "system.log" + path.write_text(content, encoding="utf-8") + return str(path) + + +def test_parse_full_scan_seconds_aws_shipper_format(tmp_path): + log = _write_log(tmp_path, AWS_SCAN_LINES) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_10m_20tok_0") == pytest.approx(66.00668) + + +def test_parse_full_scan_seconds_docker_raw_format(tmp_path): + log = _write_log(tmp_path, DOCKER_SCAN_LINES) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_local_tiny_0") == pytest.approx(2.561197) + + +def test_parse_full_scan_seconds_matches_case_insensitively(tmp_path): + """Scylla folds the index name, so the caller's un-folded name must still match.""" + log = _write_log(tmp_path, AWS_SCAN_LINES) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_10M_20tok_0") == pytest.approx(66.00668) + + +def test_parse_full_scan_seconds_ignores_other_indexes(tmp_path): + log = _write_log(tmp_path, DOCKER_SCAN_LINES + AWS_SCAN_LINES) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_local_tiny_0") == pytest.approx(2.561197) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_10m_20tok_0") == pytest.approx(66.00668) + + +def test_parse_full_scan_seconds_none_without_a_finish(tmp_path): + log = _write_log(tmp_path, DOCKER_SCAN_LINES.splitlines(keepends=True)[0]) + assert parse_full_scan_seconds(log, "fts_bench.fts_idx_local_tiny_0") is None + + +def test_parse_full_scan_seconds_none_for_unknown_index(tmp_path): + log = _write_log(tmp_path, DOCKER_SCAN_LINES) + assert parse_full_scan_seconds(log, "fts_bench.nope") is None + + +def test_parse_full_scan_seconds_none_when_log_missing(tmp_path): + assert parse_full_scan_seconds(str(tmp_path / "absent.log"), "fts_bench.idx") is None + + +def test_index_key_folds_case(): + assert index_key("fts_bench", "fts_idx_10M_20tok_0") == "fts_bench.fts_idx_10m_20tok_0" + + +def test_wait_for_index_build_seconds_returns_the_measurement(tmp_path): + log = _write_log(tmp_path, DOCKER_SCAN_LINES) + assert wait_for_index_build_seconds(log, "fts_bench", "fts_idx_local_tiny_0") == pytest.approx(2.561197) + + +def test_wait_for_index_build_seconds_retries_until_the_lines_are_shipped(tmp_path, monkeypatch): + """The lines are written on the node and forwarded asynchronously, so an empty log means + 'not yet', not 'never'.""" + log = _write_log(tmp_path, "") + monkeypatch.setattr(vector_store_index.time, "sleep", lambda _seconds: _write_log(tmp_path, DOCKER_SCAN_LINES)) + + assert wait_for_index_build_seconds(log, "fts_bench", "fts_idx_local_tiny_0", timeout=10) == pytest.approx(2.561197) + + +def test_wait_for_index_build_seconds_gives_up_and_returns_none(tmp_path): + """A missing measurement, not a failed build: the index is queryable either way.""" + log = _write_log(tmp_path, "") + assert wait_for_index_build_seconds(log, "fts_bench", "idx", timeout=0) is None + + +def test_index_build_columns_name_the_count_in_the_workloads_own_words(): + columns = index_build_columns("document_count", "docs") + assert [column.name for column in columns] == ["build_time", "document_count", "indexing_throughput"] + assert [column.unit for column in columns] == ["s", "docs", "docs/s"] + + +class _RecordingTable: + """Stands in for the workload's Argus table, keeping what was written to it.""" + + def __init__(self): + self.rows = {} + + def add_result(self, column, row, value, status): + self.rows.setdefault(row, {})[column] = value + + +def _send(monkeypatch, build_time, count, count_column="document_count"): + submitted = [] + monkeypatch.setattr(vector_store_index, "submit_results_to_argus", lambda client, table: submitted.append(table)) + table = _RecordingTable() + send_index_build_result( + argus_client=object(), + result_table=table, + count_column=count_column, + build_time=build_time, + count=count, + row_key="local_tiny | 1,000 docs | build #1", + ) + return table, submitted + + +def test_send_index_build_result_reports_the_throughput_the_two_others_imply(monkeypatch): + """The throughput column is the only derived value in the row, and Argus keeps its history -- + a wrong one is indistinguishable from a real regression.""" + table, submitted = _send(monkeypatch, build_time=2.5, count=1000) + assert table.rows == { + "local_tiny | 1,000 docs | build #1": { + "build_time": 2.5, + "document_count": 1000, + "indexing_throughput": 400.0, + } + } + assert submitted == [table] + + +@pytest.mark.parametrize("build_time, count", ((0.0, 1000), (2.5, 0))) +def test_send_index_build_result_reports_zero_throughput_it_cannot_derive(monkeypatch, build_time, count): + """Rather than dividing by zero, or reporting a throughput no documents were indexed at.""" + table, submitted = _send(monkeypatch, build_time=build_time, count=count) + assert table.rows["local_tiny | 1,000 docs | build #1"]["indexing_throughput"] == 0.0 + assert submitted == [table] + + +def test_send_index_build_result_counts_in_the_workloads_own_column(monkeypatch): + """The count column is named by the caller, and Argus keys the history by that name.""" + table, _ = _send(monkeypatch, build_time=2.0, count=10, count_column="vector_count") + assert "vector_count" in table.rows["local_tiny | 1,000 docs | build #1"]