@@ -120,17 +120,19 @@ def make_sim_data_dict(exp_id: str, variants: list[int], sim_data_path: list[str
120120 return {exp_id : dict (zip (variants , sim_data_path ))}
121121
122122
123- def build_duckdb_filter (config : dict ) -> str :
123+ def build_duckdb_filter (config : dict ) -> list [ tuple [ str , str ]] :
124124 """
125125 Build a DuckDB WHERE clause from config filters.
126126
127127 Args:
128128 config: Configuration dictionary with filter values
129129
130130 Returns:
131- DuckDB WHERE clause string
131+ List of (column_name, condition_string) tuples representing each
132+ filter condition. Join the condition strings with " AND " to form
133+ a complete WHERE clause.
132134 """
133- duckdb_filter = []
135+ duckdb_filter : list [ tuple [ str , str ]] = []
134136 last_analysis_level = - 1
135137 filter_types = list (FILTERS .keys ())
136138
@@ -166,20 +168,28 @@ def build_duckdb_filter(config: dict) -> str:
166168 if len (config [data_filter ]) > 1 :
167169 if data_type is str :
168170 filter_values = "', '" .join (str (i ) for i in config [data_filter ])
169- duckdb_filter .append (f"{ data_filter } IN ('{ filter_values } ')" )
171+ duckdb_filter .append (
172+ (data_filter , f"{ data_filter } IN ('{ filter_values } ')" )
173+ )
170174 else :
171175 filter_values = ", " .join (str (i ) for i in config [data_filter ])
172- duckdb_filter .append (f"{ data_filter } IN ({ filter_values } )" )
176+ duckdb_filter .append (
177+ (data_filter , f"{ data_filter } IN ({ filter_values } )" )
178+ )
173179 else :
174180 if data_type is str :
175181 quoted_val = str (config [data_filter ][0 ])
176- duckdb_filter .append (f"{ data_filter } = '{ quoted_val } '" )
182+ duckdb_filter .append (
183+ (data_filter , f"{ data_filter } = '{ quoted_val } '" )
184+ )
177185 else :
178- duckdb_filter .append (f"{ data_filter } = { config [data_filter ][0 ]} " )
186+ duckdb_filter .append (
187+ (data_filter , f"{ data_filter } = { config [data_filter ][0 ]} " )
188+ )
179189
180190 last_analysis_level = current_analysis_level
181191
182- return " AND " . join ( duckdb_filter )
192+ return duckdb_filter
183193
184194
185195def load_variant_metadata (
@@ -290,7 +300,7 @@ def filter_variant_dicts(
290300
291301def build_query_strings (
292302 analysis_type : str ,
293- duckdb_filter : str ,
303+ duckdb_filter : list [ tuple [ str , str ]] ,
294304 config_sql : str ,
295305 history_sql : str ,
296306 success_sql : str ,
@@ -302,7 +312,8 @@ def build_query_strings(
302312
303313 Args:
304314 analysis_type: Type of analysis (e.g., "multivariant", "single")
305- duckdb_filter: DuckDB WHERE clause
315+ duckdb_filter: List of (column_name, condition_string) tuples from
316+ build_duckdb_filter
306317 config_sql: SQL query for config data
307318 history_sql: SQL query for history data
308319 success_sql: SQL query for success data
@@ -314,6 +325,7 @@ def build_query_strings(
314325 (history_query, config_query, success_query, output_dir, variant_set)
315326 """
316327 id_cols = ANALYSIS_TYPES [analysis_type ]
328+ duckdb_filter_str = " AND " .join (cond for _ , cond in duckdb_filter )
317329 query_strings = {}
318330
319331 if len (id_cols ) > 0 :
@@ -326,7 +338,7 @@ def build_query_strings(
326338 select_cols = ", " .join (cols_to_select )
327339
328340 data_ids_with_variants = conn .sql (
329- f"SELECT DISTINCT { select_cols } FROM ({ config_sql } ) WHERE { duckdb_filter } "
341+ f"SELECT DISTINCT { select_cols } FROM ({ config_sql } ) WHERE { duckdb_filter_str } "
330342 ).fetchall ()
331343
332344 # Group by the id_cols to collect variants for each subset
@@ -339,6 +351,12 @@ def build_query_strings(
339351 var_id = row [var_id_idx ]
340352 id_to_variants [data_id ].add ((exp_id , var_id ))
341353
354+ # Extract lower-level filter conditions (columns not in id_cols) so
355+ # they are still applied in the per-subset SQL passed to analysis scripts.
356+ lower_level_filter = " AND " .join (
357+ cond for col , cond in duckdb_filter if col not in id_cols
358+ )
359+
342360 for data_id , variant_set in id_to_variants .items ():
343361 data_filters = []
344362 curr_outdir = os .path .abspath (outdir )
@@ -350,10 +368,15 @@ def build_query_strings(
350368 data_filters .append (f"{ col } ={ col_val } " )
351369 os .makedirs (curr_outdir , exist_ok = True )
352370 data_filters = " AND " .join (data_filters )
371+ full_filter = (
372+ f"{ data_filters } AND { lower_level_filter } "
373+ if lower_level_filter
374+ else data_filters
375+ )
353376 query_strings [data_filters ] = (
354- f"SELECT * FROM ({ history_sql } ) WHERE { data_filters } " ,
355- f"SELECT * FROM ({ config_sql } ) WHERE { data_filters } " ,
356- f"SELECT * FROM ({ success_sql } ) WHERE { data_filters } " ,
377+ f"SELECT * FROM ({ history_sql } ) WHERE { full_filter } " ,
378+ f"SELECT * FROM ({ config_sql } ) WHERE { full_filter } " ,
379+ f"SELECT * FROM ({ success_sql } ) WHERE { full_filter } " ,
357380 curr_outdir ,
358381 variant_set ,
359382 )
@@ -363,13 +386,13 @@ def build_query_strings(
363386 # For analysis types with no id_cols, query all variants matching the filter
364387 all_variants = conn .sql (
365388 f"SELECT DISTINCT experiment_id, variant"
366- f" FROM ({ config_sql } ) WHERE { duckdb_filter } "
389+ f" FROM ({ config_sql } ) WHERE { duckdb_filter_str } "
367390 ).fetchall ()
368391 variant_set = set (all_variants )
369- query_strings [duckdb_filter ] = (
370- f"SELECT * FROM ({ history_sql } ) WHERE { duckdb_filter } " ,
371- f"SELECT * FROM ({ config_sql } ) WHERE { duckdb_filter } " ,
372- f"SELECT * FROM ({ success_sql } ) WHERE { duckdb_filter } " ,
392+ query_strings [duckdb_filter_str ] = (
393+ f"SELECT * FROM ({ history_sql } ) WHERE { duckdb_filter_str } " ,
394+ f"SELECT * FROM ({ config_sql } ) WHERE { duckdb_filter_str } " ,
395+ f"SELECT * FROM ({ success_sql } ) WHERE { duckdb_filter_str } " ,
373396 os .path .abspath (outdir ),
374397 variant_set ,
375398 )
@@ -383,7 +406,7 @@ def run_analysis_loop(
383406 history_sql : str ,
384407 config_sql : str ,
385408 success_sql : str ,
386- duckdb_filter : str ,
409+ duckdb_filter : list [ tuple [ str , str ]] ,
387410 variant_metadata : dict ,
388411 sim_data_dict : dict ,
389412 variant_names : dict ,
@@ -397,7 +420,8 @@ def run_analysis_loop(
397420 history_sql: SQL query for history data
398421 config_sql: SQL query for config data
399422 success_sql: SQL query for success data
400- duckdb_filter: DuckDB WHERE clause for filtering data
423+ duckdb_filter: List of (column_name, condition_string) tuples from
424+ build_duckdb_filter
401425 variant_metadata: Variant metadata dictionary
402426 sim_data_dict: Sim data dictionary
403427 variant_names: Variant names dictionary
0 commit comments