diff --git a/Cargo.lock b/Cargo.lock index a1c90ec..48b5d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "cc" version = "1.0.79" @@ -26,6 +32,88 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "httpdate" version = "1.0.2" @@ -38,6 +126,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "jwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56" +dependencies = [ + "crossbeam", + "rayon", +] + [[package]] name = "libc" version = "0.2.139" @@ -53,6 +151,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "pkg-config" version = "0.3.26" @@ -77,12 +194,40 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "ryu" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.152" @@ -118,6 +263,7 @@ dependencies = [ name = "seroost" version = "0.1.0" dependencies = [ + "jwalk", "serde", "serde_json", "sqlite", diff --git a/Cargo.toml b/Cargo.toml index 8e8945d..5d5702c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +jwalk = "0.8.1" serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.91" sqlite = "0.30.3" diff --git a/src/main.rs b/src/main.rs index 08131f3..a1e78dc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,36 +1,47 @@ +use jwalk::rayon::prelude::*; +use std::env; use std::fs::{self, File}; +use std::io::{BufReader, BufWriter}; use std::path::Path; -use xml::reader::{XmlEvent, EventReader}; -use xml::common::{Position, TextPosition}; -use std::env; -use std::result::Result; use std::process::ExitCode; +use std::result::Result; use std::str; -use std::io::{BufReader, BufWriter}; +use std::sync::atomic::AtomicUsize; +use xml::common::{Position, TextPosition}; +use xml::reader::{EventReader, XmlEvent}; mod model; use model::*; -mod server; mod lexer; +mod server; pub mod snowball; fn parse_entire_txt_file(file_path: &Path) -> Result { fs::read_to_string(file_path).map_err(|err| { - eprintln!("ERROR: coult not open file {file_path}: {err}", file_path = file_path.display()); + eprintln!( + "ERROR: coult not open file {file_path}: {err}", + file_path = file_path.display() + ); }) } fn parse_entire_xml_file(file_path: &Path) -> Result { let file = File::open(file_path).map_err(|err| { - eprintln!("ERROR: could not open file {file_path}: {err}", file_path = file_path.display()); + eprintln!( + "ERROR: could not open file {file_path}: {err}", + file_path = file_path.display() + ); })?; let er = EventReader::new(BufReader::new(file)); let mut content = String::new(); for event in er.into_iter() { let event = event.map_err(|err| { - let TextPosition {row, column} = err.position(); + let TextPosition { row, column } = err.position(); let msg = err.msg(); - eprintln!("{file_path}:{row}:{column}: ERROR: {msg}", file_path = file_path.display()); + eprintln!( + "{file_path}:{row}:{column}: ERROR: {msg}", + file_path = file_path.display() + ); })?; if let XmlEvent::Characters(text) = event { @@ -42,18 +53,25 @@ fn parse_entire_xml_file(file_path: &Path) -> Result { } fn parse_entire_file_by_extension(file_path: &Path) -> Result { - let extension = file_path.extension().ok_or_else(|| { - eprintln!("ERROR: can't detect file type of {file_path} without extension", - file_path = file_path.display()); - })?.to_string_lossy(); + let extension = file_path + .extension() + .ok_or_else(|| { + eprintln!( + "ERROR: can't detect file type of {file_path} without extension", + file_path = file_path.display() + ); + })? + .to_string_lossy(); match extension.as_ref() { "xhtml" | "xml" => parse_entire_xml_file(file_path), // TODO: specialized parser for markdown files "txt" | "md" => parse_entire_txt_file(file_path), _ => { - eprintln!("ERROR: can't detect file type of {file_path}: unsupported extension {extension}", - file_path = file_path.display(), - extension = extension); + eprintln!( + "ERROR: can't detect file type of {file_path}: unsupported extension {extension}", + file_path = file_path.display(), + extension = extension + ); Err(()) } } @@ -73,45 +91,50 @@ fn save_model_as_json(model: &InMemoryModel, index_path: &str) -> Result<(), ()> Ok(()) } -fn add_folder_to_model(dir_path: &Path, model: &mut dyn Model, skipped: &mut usize) -> Result<(), ()> { - let dir = fs::read_dir(dir_path).map_err(|err| { - eprintln!("ERROR: could not open directory {dir_path} for indexing: {err}", - dir_path = dir_path.display()); - })?; - - 'next_file: for file in dir { - let file = file.map_err(|err| { - eprintln!("ERROR: could not read next file in directory {dir_path} during indexing: {err}", - dir_path = dir_path.display()); - })?; +fn add_folder_to_model( + dir_path: &Path, + model: &mut dyn Model, + skipped: &mut AtomicUsize, +) -> Result<(), ()> { + let dir: Vec<_> = jwalk::WalkDir::new(dir_path).into_iter().collect(); + let documents: Vec<_> = dir + .into_par_iter() + .filter_map(|file| { + let file = match file { + Ok(file) => file, + Err(err) => { + eprintln!( + "ERROR: could not read next file in directory {dir_path} during indexing: {err}", + dir_path = dir_path.display()); + return None; + } + }; - let file_path = file.path(); + if file.file_type().is_dir() { + return None; + }; - let file_type = file.file_type().map_err(|err| { - eprintln!("ERROR: could not determine type of file {file_path}: {err}", - file_path = file_path.display()); - })?; + let file_path = file.path(); - if file_type.is_dir() { - add_folder_to_model(&file_path, model, skipped)?; - continue 'next_file; - } + // TODO: how does this work with symlinks? - // TODO: how does this work with symlinks? + println!("Indexing {:?}...", &file_path); - println!("Indexing {:?}...", &file_path); + let content = match parse_entire_file_by_extension(&file_path) { + Ok(content) => content.chars().collect::>(), + Err(()) => { + skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return None; + } + }; - let content = match parse_entire_file_by_extension(&file_path) { - Ok(content) => content.chars().collect::>(), - Err(()) => { - *skipped += 1; - continue 'next_file; - } - }; + return Some((file_path, content)); + }) + .collect(); - model.add_document(file_path, &content)?; + for (path, content) in documents { + model.add_document(path, &content)?; } - Ok(()) } @@ -135,7 +158,7 @@ fn entry() -> Result<(), ()> { "--sqlite" => use_sqlite_mode = true, _ => { subcommand = Some(arg); - break + break; } } } @@ -152,7 +175,7 @@ fn entry() -> Result<(), ()> { eprintln!("ERROR: no directory is provided for {subcommand} subcommand"); })?; - let mut skipped = 0; + let mut skipped = AtomicUsize::new(0); if use_sqlite_mode { let index_path = "index.db"; @@ -160,7 +183,7 @@ fn entry() -> Result<(), ()> { if let Err(err) = fs::remove_file(index_path) { if err.kind() != std::io::ErrorKind::NotFound { eprintln!("ERROR: could not delete file {index_path}: {err}"); - return Err(()) + return Err(()); } } @@ -176,19 +199,26 @@ fn entry() -> Result<(), ()> { save_model_as_json(&model, index_path)?; } - println!("Skipped {skipped} files."); + println!( + "Skipped {} files.", + skipped.load(std::sync::atomic::Ordering::SeqCst) + ); Ok(()) - }, + } "search" => { let index_path = args.next().ok_or_else(|| { usage(&program); eprintln!("ERROR: no path to index is provided for {subcommand} subcommand"); })?; - let prompt = args.next().ok_or_else(|| { - usage(&program); - eprintln!("ERROR: no search query is provided {subcommand} subcommand"); - })?.chars().collect::>(); + let prompt = args + .next() + .ok_or_else(|| { + usage(&program); + eprintln!("ERROR: no search query is provided {subcommand} subcommand"); + })? + .chars() + .collect::>(); if use_sqlite_mode { let model = SqliteModel::open(Path::new(&index_path))?; @@ -201,9 +231,10 @@ fn entry() -> Result<(), ()> { eprintln!("ERROR: could not open index file {index_path}: {err}"); })?; - let model = serde_json::from_reader::<_, InMemoryModel>(index_file).map_err(|err| { - eprintln!("ERROR: could not parse index file {index_path}: {err}"); - })?; + let model = + serde_json::from_reader::<_, InMemoryModel>(index_file).map_err(|err| { + eprintln!("ERROR: could not parse index file {index_path}: {err}"); + })?; for (path, rank) in model.search_query(&prompt)?.iter().take(20) { println!("{path} {rank}", path = path.display());