Feature entry correction

R/data_processing.R

@@ -446,6 +446,7 @@ NULL
     tidyr::separate_rows(protein_ids, sep = ";") |>
     # dplyr::filter(!stringr::str_detect(protein_ids, "_pseudo")) |>
     dplyr::mutate(protein_ids = gsub("_pseudo", "", protein_ids)) |>
+    dplyr::mutate(protein_ids = gsub("_len", "", protein_ids)) |>
     DBI::dbWriteTable(conn = con, name = "genome_gene_protein", overwrite = TRUE)
 }
 

R/feature_to_head.R

@@ -411,7 +411,7 @@ buildDyadFeatureMap <- function(
       view_name = "v_pfam",
       parquet_dir = parquet_dir,
       dataset_name = "protein_Pfam",
-      feature_expr = "query_name"
+      feature_expr = "REPLACE(query_name, '-', '.')"
     )
   }
 
@@ -444,7 +444,7 @@ buildDyadFeatureMap <- function(
       view_name = "v_defensecas",
       parquet_dir = parquet_dir,
       dataset_name = "protein_DefenseCas",
-      feature_expr = "query_name"
+      feature_expr = "REPLACE(query_name, '-', '.')"
     )
   }
 

vignettes/BVBRC_stats.Rmd

@@ -298,6 +298,27 @@ clean_all_char_cols <- function(df) {
     ))
 }
 
+# Clean ALL List columns in the data frame
+clean_list_col <- function(x) {
+  purrr::map(
+    x,
+    \(z) {
+      if (is.null(z) || length(z) == 0)
+        return(NA_character_)
+
+      z <- as.character(unlist(z))
+      z <- stringr::str_squish(z)
+
+      z[is_placeholder_na(z)] <- NA_character_
+
+      if (all(is.na(z)))
+        return(NA_character_)
+
+      z
+    }
+  )
+}
+
 # --- Block-level summaries ---------------------------------------------------
 summarize_block <- function(df, cols, block_name) {
   if (length(cols) == 0) {
@@ -337,6 +358,40 @@ bvbrc <- fetchCompleteBVBRCMetadataAPI()
 
 # Apply to your table
 bvbrc_clean <- clean_all_char_cols(bvbrc)
+list_cols <- names(bvbrc_clean)[sapply(bvbrc_clean, is.list)]
+bvbrc_clean <- bvbrc_clean |>
+  mutate(
+    across(all_of(list_cols), clean_list_col)
+  )
+
+# Save the table to parquet
+
+flatten_list <- function(x) {
+  purrr::map_chr(
+    x,
+    \(z) {
+      if (is.null(z) || length(z) == 0)
+        return(NA_character_)
+
+      paste(as.character(unlist(z)), collapse = ";")
+    }
+  )
+}
+
+bvbrc_clean <- bvbrc_clean |>
+  dplyr::mutate(
+    dplyr::across(
+      dplyr::all_of(list_cols),
+      flatten_list
+    )
+  )
+
+arrow::write_parquet(
+  bvbrc_clean,
+  "data/bvbrc_clean.parquet",
+  compression = "zstd"
+)
+
 # --- Per-column stats --------------------------------------------------------
 col_stats <- tibble::tibble(column = colnames(bvbrc_clean)) |>
   dplyr::mutate(