chore: Make the repo use examples and keep the library

examples/csv-indexer.rs

@@ -0,0 +1,148 @@
+#[macro_use] extern crate serde_derive;
+
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+use std::fs::File;
+use std::io;
+
+use csv::ReaderBuilder;
+use raptor::{MetadataBuilder, DocIndex, Tokenizer, CommonWords};
+use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct CommandCsv {
+    /// The stop word file, each word must be separated by a newline.
+    #[structopt(long = "stop-words", parse(from_os_str))]
+    pub stop_words: PathBuf,
+
+    /// The csv file to index.
+    #[structopt(parse(from_os_str))]
+    pub products: PathBuf,
+}
+
+#[derive(Debug, Deserialize)]
+struct Product {
+    id: u64,
+    title: String,
+    description: String,
+    image: String,
+}
+
+#[derive(Debug)]
+pub struct CsvIndexer {
+    common_words: CommonWords,
+    products: PathBuf,
+}
+
+impl CsvIndexer {
+    pub fn from_command(command: CommandCsv) -> io::Result<CsvIndexer> {
+        let common_words = CommonWords::from_file(command.stop_words)?;
+        let products = command.products;
+
+        Ok(CsvIndexer { common_words, products })
+    }
+
+    pub fn index(self) {
+        let random_name = PathBuf::from(moby_name_gen::random_name());
+        let map_file = random_name.with_extension("map");
+        let idx_file = random_name.with_extension("idx");
+        let sst_file = random_name.with_extension("sst");
+
+        let env_options = EnvOptions::new();
+        let cf_options = ColumnFamilyOptions::new();
+        let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
+        let sst_file = sst_file.to_str().unwrap();
+        sst_file_writer.open(&sst_file).expect("open the sst file");
+
+        let map = File::create(&map_file).unwrap();
+        let indexes = File::create(&idx_file).unwrap();
+        let mut builder = MetadataBuilder::new(map, indexes);
+        let mut fields = BTreeMap::new();
+
+        let mut rdr = ReaderBuilder::new().from_path(&self.products).expect("reading product file");
+        let mut errors = 0;
+
+        for result in rdr.deserialize() {
+            let product: Product = match result {
+                Ok(product) => product,
+                Err(e) => { eprintln!("{:?}", e); errors += 1; continue },
+            };
+
+            {
+                let string_id = product.id.to_string();
+                insert_document_words(&mut builder, product.id, 0, Some((0, string_id.as_str())));
+
+                let key = format!("{}-id", product.id);
+                let value = string_id;
+                fields.insert(key, value);
+            }
+
+            {
+                let title = Tokenizer::new(&product.title);
+                let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
+                insert_document_words(&mut builder, product.id, 1, title);
+
+                let key = format!("{}-title", product.id);
+                let value = product.title;
+                fields.insert(key, value);
+            }
+
+            {
+                let description = Tokenizer::new(&product.description);
+                let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
+                insert_document_words(&mut builder, product.id, 2, description);
+
+                let key = format!("{}-description", product.id);
+                let value = product.description;
+                fields.insert(key, value);
+            }
+
+            {
+                let key = format!("{}-image", product.id);
+                let value = product.image;
+                fields.insert(key, value);
+            }
+        }
+
+        for (key, value) in fields {
+            sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
+        }
+        let _sst_file_info = sst_file_writer.finish().unwrap();
+
+        builder.finish().unwrap();
+
+        println!("Found {} errorneous lines", errors);
+        println!("Succesfully created {:?} dump.", random_name);
+    }
+}
+
+fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
+where A: io::Write,
+      B: io::Write,
+      I: IntoIterator<Item=(usize, &'a str)>,
+{
+    for (index, word) in words {
+        let doc_index = DocIndex {
+            document: doc_index,
+            attribute: attr,
+            attribute_index: index as u32,
+        };
+        // insert the exact representation
+        let word_lower = word.to_lowercase();
+
+        // and the unidecoded lowercased version
+        let word_unidecoded = unidecode::unidecode(word).to_lowercase();
+        if word_lower != word_unidecoded {
+            builder.insert(word_unidecoded, doc_index);
+        }
+
+        builder.insert(word_lower, doc_index);
+    }
+}
+
+fn main() {
+    let command = CommandCsv::from_args();
+    let indexer = CsvIndexer::from_command(command).unwrap();
+    indexer.index();
+}

examples/json-lines-indexer.rs

@@ -0,0 +1,153 @@
+#[macro_use] extern crate serde_derive;
+
+use std::collections::BTreeMap;
+use std::io::{self, BufReader, BufRead};
+use std::fs::File;
+use std::path::PathBuf;
+
+use serde_json::from_str;
+use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
+use raptor::{MetadataBuilder, DocIndex, Tokenizer, CommonWords};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct CommandJsonLines {
+    /// The stop word file, each word must be separated by a newline.
+    #[structopt(long = "stop-words", parse(from_os_str))]
+    pub stop_words: PathBuf,
+
+    /// The csv file to index.
+    #[structopt(parse(from_os_str))]
+    pub products: PathBuf,
+}
+
+#[derive(Debug, Deserialize)]
+struct Product {
+    id: u64,
+    title: String,
+    description: String,
+    image: String,
+}
+
+#[derive(Debug)]
+pub struct JsonLinesIndexer {
+    common_words: CommonWords,
+    products: PathBuf,
+}
+
+impl JsonLinesIndexer {
+    pub fn from_command(command: CommandJsonLines) -> io::Result<JsonLinesIndexer> {
+        let common_words = CommonWords::from_file(command.stop_words)?;
+        let products = command.products;
+
+        Ok(JsonLinesIndexer { common_words, products })
+    }
+
+    pub fn index(self) {
+        let data = File::open(&self.products).unwrap();
+        let data = BufReader::new(data);
+
+        // TODO add a subcommand to pack these files in a tar.xxx archive
+        let random_name = PathBuf::from(moby_name_gen::random_name());
+        let map_file = random_name.with_extension("map");
+        let idx_file = random_name.with_extension("idx");
+        let sst_file = random_name.with_extension("sst");
+
+        let env_options = EnvOptions::new();
+        let cf_options = ColumnFamilyOptions::new();
+        let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
+        let sst_file = sst_file.to_str().unwrap();
+        sst_file_writer.open(&sst_file).expect("open the sst file");
+
+        let map = File::create(&map_file).unwrap();
+        let indexes = File::create(&idx_file).unwrap();
+        let mut builder = MetadataBuilder::new(map, indexes);
+        let mut fields = BTreeMap::new();
+        let mut errors = 0;
+
+        for result in data.lines() {
+            let product: Product = match result {
+                Ok(product) => match from_str(&product) {
+                    Ok(product) => product,
+                    Err(e) => { eprintln!("{:?}", e); errors += 1; continue },
+                },
+                Err(e) => { eprintln!("{:?}", e); errors += 1; continue },
+            };
+
+            {
+                let string_id = product.id.to_string();
+                insert_document_words(&mut builder, product.id, 0, Some((0, string_id.as_str())));
+
+                let key = format!("{}-id", product.id);
+                let value = string_id;
+                fields.insert(key, value);
+            }
+
+            {
+                let title = Tokenizer::new(&product.title);
+                let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
+                insert_document_words(&mut builder, product.id, 1, title);
+
+                let key = format!("{}-title", product.id);
+                let value = product.title;
+                fields.insert(key, value);
+            }
+
+            {
+                let description = Tokenizer::new(&product.description);
+                let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
+                insert_document_words(&mut builder, product.id, 2, description);
+
+                let key = format!("{}-description", product.id);
+                let value = product.description;
+                fields.insert(key, value);
+            }
+
+            {
+                let key = format!("{}-image", product.id);
+                let value = product.image;
+                fields.insert(key, value);
+            }
+        }
+
+        for (key, value) in fields {
+            sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
+        }
+        let _sst_file_info = sst_file_writer.finish().unwrap();
+
+        builder.finish().unwrap();
+
+        println!("Found {} errorneous lines", errors);
+        println!("Succesfully created {:?} dump.", random_name);
+    }
+}
+
+fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
+where A: io::Write,
+      B: io::Write,
+      I: IntoIterator<Item=(usize, &'a str)>,
+{
+    for (index, word) in words {
+        let doc_index = DocIndex {
+            document: doc_index,
+            attribute: attr,
+            attribute_index: index as u32,
+        };
+        // insert the exact representation
+        let word_lower = word.to_lowercase();
+
+        // and the unidecoded lowercased version
+        let word_unidecoded = unidecode::unidecode(word).to_lowercase();
+        if word_lower != word_unidecoded {
+            builder.insert(word_unidecoded, doc_index);
+        }
+
+        builder.insert(word_lower, doc_index);
+    }
+}
+
+fn main() {
+    let command = CommandJsonLines::from_args();
+    let indexer = JsonLinesIndexer::from_command(command).unwrap();
+    indexer.index();
+}

examples/serve-console.rs

@@ -0,0 +1,90 @@
+use std::str::from_utf8_unchecked;
+use std::io::{self, Write};
+use structopt::StructOpt;
+use std::path::PathBuf;
+
+use fst::Streamer;
+use elapsed::measure_time;
+use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
+use raptor::{automaton, Metadata, RankedStream, CommonWords};
+
+#[derive(Debug, StructOpt)]
+pub struct CommandConsole {
+    /// The stop word file, each word must be separated by a newline.
+    #[structopt(long = "stop-words", parse(from_os_str))]
+    pub stop_words: PathBuf,
+
+    /// Meta file name (e.g. relaxed-colden).
+    #[structopt(parse(from_os_str))]
+    pub meta_name: PathBuf,
+}
+
+pub struct ConsoleSearch {
+    common_words: CommonWords,
+    metadata: Metadata,
+    db: DB,
+}
+
+impl ConsoleSearch {
+    pub fn from_command(command: CommandConsole) -> io::Result<ConsoleSearch> {
+        let common_words = CommonWords::from_file(command.stop_words)?;
+
+        let map_file = command.meta_name.with_extension("map");
+        let idx_file = command.meta_name.with_extension("idx");
+        let sst_file = command.meta_name.with_extension("sst");
+
+        let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
+
+        let rocksdb = "rocksdb/storage";
+        let db = DB::open_default(rocksdb).unwrap();
+        let sst_file = sst_file.to_str().unwrap();
+        db.ingest_external_file(&IngestExternalFileOptions::new(), &[sst_file]).unwrap();
+        drop(db);
+        let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
+
+        Ok(ConsoleSearch { common_words, metadata, db })
+    }
+
+    pub fn serve(self) {
+        loop {
+            print!("Searching for: ");
+            io::stdout().flush().unwrap();
+
+            let mut query = String::new();
+            io::stdin().read_line(&mut query).unwrap();
+            let query = query.trim().to_lowercase();
+
+            if query.is_empty() { break }
+
+            let (elapsed, _) = measure_time(|| search(&self.metadata, &self.db, &self.common_words, &query));
+            println!("Finished in {}", elapsed);
+        }
+    }
+}
+
+fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) {
+    let mut automatons = Vec::new();
+    for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) {
+        let lev = automaton::build(query);
+        automatons.push(lev);
+    }
+
+    let mut stream = RankedStream::new(&metadata, automatons, 20);
+    while let Some(document) = stream.next() {
+        let id_key = format!("{}-id", document.document_id);
+        let id = database.get(id_key.as_bytes()).unwrap().unwrap();
+        let id = unsafe { from_utf8_unchecked(&id) };
+        print!("{} ", id);
+
+        let title_key = format!("{}-title", document.document_id);
+        let title = database.get(title_key.as_bytes()).unwrap().unwrap();
+        let title = unsafe { from_utf8_unchecked(&title) };
+        println!("{:?}", title);
+    }
+}
+
+fn main() {
+    let command = CommandConsole::from_args();
+    let console = ConsoleSearch::from_command(command).unwrap();
+    console.serve()
+}

examples/serve-http.rs

@@ -0,0 +1,143 @@
+#[macro_use] extern crate serde_derive;
+
+use std::str::from_utf8_unchecked;
+use std::io::{self, Write};
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::error::Error;
+use std::sync::Arc;
+
+use raptor::rank::RankedStream;
+use raptor::{automaton, Metadata, CommonWords};
+use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
+use fst::Streamer;
+use warp::Filter;
+
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct CommandHttp {
+    /// The address and port to bind the server to.
+    #[structopt(short = "l", default_value = "127.0.0.1:3030")]
+    pub listen_addr: SocketAddr,
+
+    /// The stop word file, each word must be separated by a newline.
+    #[structopt(long = "stop-words", parse(from_os_str))]
+    pub stop_words: PathBuf,
+
+    /// Meta file name (e.g. relaxed-colden).
+    #[structopt(parse(from_os_str))]
+    pub meta_name: PathBuf,
+}
+
+#[derive(Debug, Serialize)]
+struct Document<'a> {
+    id: u64,
+    title: &'a str,
+    description: &'a str,
+    image: &'a str,
+}
+
+#[derive(Debug, Deserialize)]
+struct SearchQuery { q: String }
+
+pub struct HttpServer {
+    listen_addr: SocketAddr,
+    common_words: Arc<CommonWords>,
+    metadata: Arc<Metadata>,
+    db: Arc<DB>,
+}
+
+impl HttpServer {
+    pub fn from_command(command: CommandHttp) -> io::Result<HttpServer> {
+        let common_words = CommonWords::from_file(command.stop_words)?;
+
+        let map_file = command.meta_name.with_extension("map");
+        let idx_file = command.meta_name.with_extension("idx");
+        let sst_file = command.meta_name.with_extension("sst");
+        let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
+
+        let rocksdb = "rocksdb/storage";
+        let db = DB::open_default(rocksdb).unwrap();
+        let sst_file = sst_file.to_str().unwrap();
+        db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
+        drop(db);
+        let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
+
+        Ok(HttpServer {
+            listen_addr: command.listen_addr,
+            common_words: Arc::new(common_words),
+            metadata: Arc::new(metadata),
+            db: Arc::new(db),
+        })
+    }
+
+    pub fn serve(self) {
+        let HttpServer { listen_addr, common_words, metadata, db } = self;
+
+        let routes = warp::path("search")
+            .and(warp::query())
+            .map(move |query: SearchQuery| {
+                let body = search(metadata.clone(), db.clone(), common_words.clone(), &query.q).unwrap();
+                body
+            })
+            .with(warp::reply::with::header("Content-Type", "application/json"))
+            .with(warp::reply::with::header("Access-Control-Allow-Origin", "*"));
+
+        warp::serve(routes).run(listen_addr)
+    }
+}
+
+fn search<M, D, C>(metadata: M, database: D, common_words: C, query: &str) -> Result<String, Box<Error>>
+where M: AsRef<Metadata>,
+      D: AsRef<DB>,
+      C: AsRef<CommonWords>,
+{
+    let mut automatons = Vec::new();
+    for query in query.split_whitespace().map(str::to_lowercase) {
+        if common_words.as_ref().contains(&query) { continue }
+        let lev = automaton::build(&query);
+        automatons.push(lev);
+    }
+
+    let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20);
+    let mut body = Vec::new();
+    write!(&mut body, "[")?;
+
+    let mut first = true;
+    while let Some(document) = stream.next() {
+        let title_key = format!("{}-title", document.document_id);
+        let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap();
+        let title = unsafe { from_utf8_unchecked(&title) };
+
+        let description_key = format!("{}-description", document.document_id);
+        let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap();
+        let description = unsafe { from_utf8_unchecked(&description) };
+
+        let image_key = format!("{}-image", document.document_id);
+        let image = database.as_ref().get(image_key.as_bytes()).unwrap().unwrap();
+        let image = unsafe { from_utf8_unchecked(&image) };
+
+        let document = Document {
+            id: document.document_id,
+            title: title,
+            description: description,
+            image: image,
+        };
+
+        if !first { write!(&mut body, ",")? }
+        serde_json::to_writer(&mut body, &document)?;
+
+        first = false;
+    }
+
+    write!(&mut body, "]")?;
+
+    Ok(String::from_utf8(body)?)
+}
+
+fn main() {
+    let command = CommandHttp::from_args();
+    let server = HttpServer::from_command(command).unwrap();
+    server.serve();
+}