Skip to content

Commit d83f0f3

Browse files
authored
Merge pull request #424 from CovertLab/misc-fixes
Honor lower-level analysis filters
2 parents d75e7c7 + 1a45f6b commit d83f0f3

7 files changed

Lines changed: 273 additions & 187 deletions

File tree

‎ecoli/processes/transcript_elongation.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -62,27 +62,28 @@ class TranscriptElongation(PartitionedProcess):
6262
6363
defaults:
6464
- rnaPolymeraseElongationRateDict (dict): Array with elongation rate
65-
set points for different media environments.
65+
set points for different media environments.
6666
- rnaIds (array[str]) : array of names for each TU
6767
- rnaLengths (array[int]) : array of lengths for each TU
68-
(in nucleotides?)
68+
(in nucleotides?)
6969
- rnaSequences (2D array[int]) : Array with the nucleotide sequences
70-
of each TU. This is in the form of a 2D array where each row is a
71-
TU, and each column is a position in the TU's sequence. Nucleotides
72-
are stored as an index {0, 1, 2, 3}, and the row is padded with
73-
-1's on the right to indicate where the sequence ends.
70+
of each TU. This is in the form of a 2D array where each row is a
71+
TU, and each column is a position in the TU's sequence. Nucleotides
72+
are stored as an index {0, 1, 2, 3}, and the row is padded with
73+
-1's on the right to indicate where the sequence ends.
7474
- ntWeights (array[float]): Array of nucleotide weights
75-
- endWeight (array[float]): ???,
75+
- endWeight (array[float]): Additional mass added when an RNA
76+
transcript is completed (termination/end-group mass adjustment),
7677
- replichore_lengths (array[int]): lengths of replichores
77-
(in nucleotides?),
78+
(in nucleotides?),
7879
- is_mRNA (array[bool]): Mask for mRNAs
7980
- ppi (str): ID of PPI
8081
- inactive_RNAP (str): ID of inactive RNAP
8182
- ntp_ids list[str]: IDs of ntp's (A, C, G, U)
8283
- variable_elongation (bool): Whether to use variable elongation.
83-
False by default.
84+
False by default.
8485
- make_elongation_rates: Function to make elongation rates, of the
85-
form: lambda random, rates, timestep, variable: rates
86+
form: lambda random, rates, timestep, variable: rates
8687
"""
8788

8889
name = NAME
@@ -151,7 +152,6 @@ def __init__(self, parameters=None):
151152
self.make_elongation_rates = self.parameters["make_elongation_rates"]
152153

153154
self.polymerized_ntps = self.parameters["polymerized_ntps"]
154-
self.charged_trna_names = self.parameters["charged_trnas"]
155155

156156
# Attenuation
157157
self.trna_attenuation = self.parameters["trna_attenuation"]

‎runscripts/analysis.py‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -120,17 +120,19 @@ def make_sim_data_dict(exp_id: str, variants: list[int], sim_data_path: list[str
120120
return {exp_id: dict(zip(variants, sim_data_path))}
121121

122122

123-
def build_duckdb_filter(config: dict) -> str:
123+
def build_duckdb_filter(config: dict) -> list[tuple[str, str]]:
124124
"""
125125
Build a DuckDB WHERE clause from config filters.
126126
127127
Args:
128128
config: Configuration dictionary with filter values
129129
130130
Returns:
131-
DuckDB WHERE clause string
131+
List of (column_name, condition_string) tuples representing each
132+
filter condition. Join the condition strings with " AND " to form
133+
a complete WHERE clause.
132134
"""
133-
duckdb_filter = []
135+
duckdb_filter: list[tuple[str, str]] = []
134136
last_analysis_level = -1
135137
filter_types = list(FILTERS.keys())
136138

@@ -166,20 +168,28 @@ def build_duckdb_filter(config: dict) -> str:
166168
if len(config[data_filter]) > 1:
167169
if data_type is str:
168170
filter_values = "', '".join(str(i) for i in config[data_filter])
169-
duckdb_filter.append(f"{data_filter} IN ('{filter_values}')")
171+
duckdb_filter.append(
172+
(data_filter, f"{data_filter} IN ('{filter_values}')")
173+
)
170174
else:
171175
filter_values = ", ".join(str(i) for i in config[data_filter])
172-
duckdb_filter.append(f"{data_filter} IN ({filter_values})")
176+
duckdb_filter.append(
177+
(data_filter, f"{data_filter} IN ({filter_values})")
178+
)
173179
else:
174180
if data_type is str:
175181
quoted_val = str(config[data_filter][0])
176-
duckdb_filter.append(f"{data_filter} = '{quoted_val}'")
182+
duckdb_filter.append(
183+
(data_filter, f"{data_filter} = '{quoted_val}'")
184+
)
177185
else:
178-
duckdb_filter.append(f"{data_filter} = {config[data_filter][0]}")
186+
duckdb_filter.append(
187+
(data_filter, f"{data_filter} = {config[data_filter][0]}")
188+
)
179189

180190
last_analysis_level = current_analysis_level
181191

182-
return " AND ".join(duckdb_filter)
192+
return duckdb_filter
183193

184194

185195
def load_variant_metadata(
@@ -290,7 +300,7 @@ def filter_variant_dicts(
290300

291301
def build_query_strings(
292302
analysis_type: str,
293-
duckdb_filter: str,
303+
duckdb_filter: list[tuple[str, str]],
294304
config_sql: str,
295305
history_sql: str,
296306
success_sql: str,
@@ -302,7 +312,8 @@ def build_query_strings(
302312
303313
Args:
304314
analysis_type: Type of analysis (e.g., "multivariant", "single")
305-
duckdb_filter: DuckDB WHERE clause
315+
duckdb_filter: List of (column_name, condition_string) tuples from
316+
build_duckdb_filter
306317
config_sql: SQL query for config data
307318
history_sql: SQL query for history data
308319
success_sql: SQL query for success data
@@ -314,6 +325,7 @@ def build_query_strings(
314325
(history_query, config_query, success_query, output_dir, variant_set)
315326
"""
316327
id_cols = ANALYSIS_TYPES[analysis_type]
328+
duckdb_filter_str = " AND ".join(cond for _, cond in duckdb_filter)
317329
query_strings = {}
318330

319331
if len(id_cols) > 0:
@@ -326,7 +338,7 @@ def build_query_strings(
326338
select_cols = ", ".join(cols_to_select)
327339

328340
data_ids_with_variants = conn.sql(
329-
f"SELECT DISTINCT {select_cols} FROM ({config_sql}) WHERE {duckdb_filter}"
341+
f"SELECT DISTINCT {select_cols} FROM ({config_sql}) WHERE {duckdb_filter_str}"
330342
).fetchall()
331343

332344
# Group by the id_cols to collect variants for each subset
@@ -339,6 +351,12 @@ def build_query_strings(
339351
var_id = row[var_id_idx]
340352
id_to_variants[data_id].add((exp_id, var_id))
341353

354+
# Extract lower-level filter conditions (columns not in id_cols) so
355+
# they are still applied in the per-subset SQL passed to analysis scripts.
356+
lower_level_filter = " AND ".join(
357+
cond for col, cond in duckdb_filter if col not in id_cols
358+
)
359+
342360
for data_id, variant_set in id_to_variants.items():
343361
data_filters = []
344362
curr_outdir = os.path.abspath(outdir)
@@ -350,10 +368,15 @@ def build_query_strings(
350368
data_filters.append(f"{col}={col_val}")
351369
os.makedirs(curr_outdir, exist_ok=True)
352370
data_filters = " AND ".join(data_filters)
371+
full_filter = (
372+
f"{data_filters} AND {lower_level_filter}"
373+
if lower_level_filter
374+
else data_filters
375+
)
353376
query_strings[data_filters] = (
354-
f"SELECT * FROM ({history_sql}) WHERE {data_filters}",
355-
f"SELECT * FROM ({config_sql}) WHERE {data_filters}",
356-
f"SELECT * FROM ({success_sql}) WHERE {data_filters}",
377+
f"SELECT * FROM ({history_sql}) WHERE {full_filter}",
378+
f"SELECT * FROM ({config_sql}) WHERE {full_filter}",
379+
f"SELECT * FROM ({success_sql}) WHERE {full_filter}",
357380
curr_outdir,
358381
variant_set,
359382
)
@@ -363,13 +386,13 @@ def build_query_strings(
363386
# For analysis types with no id_cols, query all variants matching the filter
364387
all_variants = conn.sql(
365388
f"SELECT DISTINCT experiment_id, variant"
366-
f" FROM ({config_sql}) WHERE {duckdb_filter}"
389+
f" FROM ({config_sql}) WHERE {duckdb_filter_str}"
367390
).fetchall()
368391
variant_set = set(all_variants)
369-
query_strings[duckdb_filter] = (
370-
f"SELECT * FROM ({history_sql}) WHERE {duckdb_filter}",
371-
f"SELECT * FROM ({config_sql}) WHERE {duckdb_filter}",
372-
f"SELECT * FROM ({success_sql}) WHERE {duckdb_filter}",
392+
query_strings[duckdb_filter_str] = (
393+
f"SELECT * FROM ({history_sql}) WHERE {duckdb_filter_str}",
394+
f"SELECT * FROM ({config_sql}) WHERE {duckdb_filter_str}",
395+
f"SELECT * FROM ({success_sql}) WHERE {duckdb_filter_str}",
373396
os.path.abspath(outdir),
374397
variant_set,
375398
)
@@ -383,7 +406,7 @@ def run_analysis_loop(
383406
history_sql: str,
384407
config_sql: str,
385408
success_sql: str,
386-
duckdb_filter: str,
409+
duckdb_filter: list[tuple[str, str]],
387410
variant_metadata: dict,
388411
sim_data_dict: dict,
389412
variant_names: dict,
@@ -397,7 +420,8 @@ def run_analysis_loop(
397420
history_sql: SQL query for history data
398421
config_sql: SQL query for config data
399422
success_sql: SQL query for success data
400-
duckdb_filter: DuckDB WHERE clause for filtering data
423+
duckdb_filter: List of (column_name, condition_string) tuples from
424+
build_duckdb_filter
401425
variant_metadata: Variant metadata dictionary
402426
sim_data_dict: Sim data dictionary
403427
variant_names: Variant names dictionary

‎runscripts/container/Singularity‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ From: ghcr.io/astral-sh/uv@sha256:2702577da32d9c3d55c8073e07d4476cd7402c085efbf7
3232
tar -xf /repo.tar -C /vEcoli
3333
rm /repo.tar
3434
fi
35-
apt-get update && apt-get install -y gcc procps nano curl g++
35+
# No need for ca-certificates and unzip since not installing AWS CLI
36+
apt-get update && apt-get install -y gcc procps nano curl g++ \
37+
&& apt-get clean && rm -rf /var/lib/apt/lists/*
3638
cd /vEcoli
3739
# Source shared uv environment variables
3840
. /vEcoli/singularity-env.sh

0 commit comments

Comments
 (0)