diff --git a/Cargo.lock b/Cargo.lock index 9061bc73c..af6fc6534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,11 +39,11 @@ dependencies = [ [[package]] name = "base64" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -179,7 +179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "csv" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -196,7 +196,7 @@ dependencies = [ [[package]] name = "digest" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -291,52 +291,51 @@ dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "http" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "httparse" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "hyper" -version = "0.12.9" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -374,7 +373,7 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -396,7 +395,7 @@ dependencies = [ [[package]] name = "lazycell" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -440,7 +439,7 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -521,7 +520,7 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazycell 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -595,7 +594,7 @@ name = "parking_lot" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -658,7 +657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "proc-macro2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -669,7 +668,7 @@ name = "quote" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -724,55 +723,21 @@ dependencies = [ ] [[package]] -name = "raptor-http" -version = "0.1.0" -dependencies = [ - "fst 0.3.2 (git+https://github.com/Kerollmops/fst.git?branch=automaton-for-deref)", - "raptor 0.1.0", - "rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)", - "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "warp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "raptor-indexer" -version = "0.1.0" -dependencies = [ - "moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "raptor 0.1.0", - "rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)", - "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", - "unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "raptor-indexer-csv" -version = "0.1.0" -dependencies = [ - "csv 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "raptor 0.1.0", - "rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)", - "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "raptor-search" +name = "raptor-cli" version = "0.1.0" dependencies = [ + "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "fst 0.3.2 (git+https://github.com/Kerollmops/fst.git?branch=automaton-for-deref)", + "moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "raptor 0.1.0", "rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)", + "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "warp 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -808,12 +773,12 @@ dependencies = [ [[package]] name = "ryu" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "safemem" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -851,21 +816,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.26" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -875,7 +840,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -887,7 +852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -948,7 +913,7 @@ name = "structopt-derive" version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -958,7 +923,17 @@ name = "syn" version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -993,46 +968,47 @@ dependencies = [ [[package]] name = "tokio" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-current-thread 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-codec" -version = "0.1.0" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-uds 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-codec" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-current-thread" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-executor" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1044,13 +1020,13 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-io" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1060,7 +1036,7 @@ dependencies = [ [[package]] name = "tokio-reactor" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1071,26 +1047,26 @@ dependencies = [ "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-tcp" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-threadpool" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1099,18 +1075,18 @@ dependencies = [ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-timer" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1122,14 +1098,14 @@ dependencies = [ "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-uds" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1139,8 +1115,8 @@ dependencies = [ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1153,10 +1129,10 @@ name = "tungstenite" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "input_buffer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1274,24 +1250,24 @@ dependencies = [ [[package]] name = "warp" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.9 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", "scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", "serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "tungstenite 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "urlencoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1351,7 +1327,7 @@ dependencies = [ "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum base64 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "30e93c03064e7590d0466209155251b90c22e37fab1daf2771582598b5827557" -"checksum base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "85415d2594767338a74a30c1d370b2f3262ec1b4ed2d7bba5b3faf4de40467d9" +"checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blob 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "122c3fa3949d822d2a51c648db9e8105d6e75b89dc628cc366901d3d396fa4f4" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" @@ -1369,9 +1345,9 @@ dependencies = [ "checksum crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3486aefc4c0487b9cb52372c97df0a48b8c249514af1ee99703bf70d2f2ceda1" "checksum crossbeam-epoch 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "30fecfcac6abfef8771151f8be4abc9e4edc112c2bcb233314cafde2680536e9" "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" -"checksum csv 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bbb6d75aae072248e381715437855a69595e5a97d3abbf748fe7a95e65d77fa" +"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452" "checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105" -"checksum digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "5b29c278aa8fd30796bd977169e8004b4aa88cdcd2f32a6eb22bc2d5d38df94a" +"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" "checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" @@ -1386,22 +1362,22 @@ dependencies = [ "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)" = "" "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" -"checksum http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3e5765436510a2021e6917dfb87d1afa4b32d17c71353c013897fa5da29527a1" -"checksum httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b6288d7db100340ca12873fd4d08ad1b8f206a9457798dfb17c018a33fee540" -"checksum hyper 0.12.9 (registry+https://github.com/rust-lang/crates.io-index)" = "081289d17dce471c8cbc0e69a3dd073b627e08338561d1167ab620b754d9fe90" +"checksum http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "24f58e8c2d8e886055c3ead7b28793e1455270b5fb39650984c224bc538ba581" +"checksum httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83" +"checksum hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)" = "78d50abbd1790e0f4c74cb1d4a2211b439bac661d54107ad5564c55e77906762" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08173ba1e906efb6538785a8844dd496f5d34f0a2d88038e95195172fc667220" "checksum input_buffer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e1b822cc844905551931d6f81608ed5f50a79c1078a4e2b4d42dbc7c1eedfbf" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" -"checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606" +"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" -"checksum lazycell 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e26d4c411b39f0afcf2ba6fe502be90e6c9b299c952dbd86124782520a13cffd" +"checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0" "checksum levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=new-custom-fst)" = "" "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" "checksum librocksdb_sys 0.1.0 (git+https://github.com/pingcap/rust-rocksdb.git)" = "" "checksum libz-sys 1.0.18 (git+https://github.com/busyjay/libz-sys.git?branch=static-link)" = "" -"checksum lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "949826a5ccf18c1b3a7c3d57692778d21768b79e46eb9dd07bfc4c2160036c54" +"checksum lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775751a3e69bde4df9b38dd00a1b5d6ac13791e4223d4a0506577f0dd27cfb7a" "checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f" "checksum lz4-sys 1.8.0 (git+https://github.com/busyjay/lz4-rs.git?branch=adjust-build)" = "" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" @@ -1426,7 +1402,7 @@ dependencies = [ "checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b" "checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93" "checksum pkg-config 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)" = "104630aa1c83213cbc76db0703630fcb0421dac3585063be4ce9a8a2feeaa745" -"checksum proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "ee5697238f0d893c7f0ecc59c0999f18d2af85e424de441178bcacc9f9e6cf67" +"checksum proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)" = "ffe022fb8c8bd254524b0b3305906c1921fa37a84a644e29079a9e62200c3901" "checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" @@ -1436,16 +1412,16 @@ dependencies = [ "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)" = "" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c066b8e2923f05d4718a06d2622f189ff362bc642bfade6c6629f0440f3827" -"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" +"checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" "checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum sdset 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d51ad726aa3a9c4d777b35be3a4d6d5f9d6cbc0978e81c7d690d31192f263843" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)" = "22d340507cea0b7e6632900a176101fea959c7065d93ba555072da90aaaafc87" -"checksum serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)" = "234fc8b737737b148ccd625175fc6390f5e4dacfdaa543cb93a3430d984a9119" -"checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae" +"checksum serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "31569d901045afbff7a9479f793177fe9259819aff10ab4f89ef69bbc5f567fe" +"checksum serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "bb47a3d5c84320222f66d7db21157c4a7407755de41798f9b4c1c40593397b1a" "checksum serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aaed41d9fb1e2f587201b863356590c90c1157495d811430a0c0325fe8169650" "checksum sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "51b9d1f3b5de8a167ab06834a7c883bd197f2191e1dda1a22d9ccfeedbf9aded" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" @@ -1458,21 +1434,22 @@ dependencies = [ "checksum structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8e9ad6a11096cbecdcca0cc6aa403fdfdbaeda2fb3323a39c98e6a166a1e45a" "checksum structopt-derive 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4cbce8ccdc62166bd594c14396a3242bf94c337a51dbfa9be1076dd74b3db2af" "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" +"checksum syn 0.15.7 (registry+https://github.com/rust-lang/crates.io-index)" = "455a6ec9b368f8c479b0ae5494d13b22dc00990d2f00d68c9dc6a2dc4f17f210" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" -"checksum tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fbb6a6e9db2702097bfdfddcb09841211ad423b86c75b5ddaca1d62842ac492c" -"checksum tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "881e9645b81c2ce95fcb799ded2c29ffb9f25ef5bef909089a420e5961dd8ccb" -"checksum tokio-current-thread 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fdfb899688ac16f618076bd09215edbfda0fd5dfecb375b6942636cb31fa8a7" -"checksum tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "84823b932d566bc3c6aa644df4ca36cb38593c50b7db06011fd4e12e31e4047e" +"checksum tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6e93c78d23cc61aa245a8acd2c4a79c4d7fa7fb5c3ca90d5737029f043a84895" +"checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f" +"checksum tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f90fcd90952f0a496d438a976afba8e5c205fb12123f813d8ab3aa1c8436638c" +"checksum tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c117b6cf86bb730aab4834f10df96e4dd586eff2c3c27d3781348da49e255bde" "checksum tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5cbe4ca6e71cb0b62a66e4e6f53a8c06a6eefe46cc5f665ad6f274c9906f135" -"checksum tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6cc2de7725863c86ac71b0b9068476fec50834f055a243558ef1655bbd34cb" -"checksum tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4bfbaf9f260635649ec26b6fb4aded03887295ffcd999f6e43fd2c4758f758ea" -"checksum tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5b4c329b47f071eb8a746040465fa751bd95e4716e98daef6a9b4e434c17d565" -"checksum tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a5758cecb6e0633cea5d563ac07c975e04961690b946b04fd84e7d6445a8f6af" -"checksum tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d03fa701f9578a01b7014f106b47f0a363b4727a7f3f75d666e312ab7acbbf1c" +"checksum tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8a85fffbec3c5ab1ab62324570230dcd37ee5996a7859da5caf7b9d45e3e8c" +"checksum tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4b26fd37f1125738b2170c80b551f69ff6fecb277e6e5ca885e53eec2b005018" +"checksum tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7ad235e9dadd126b2d47f6736f65aa1fdcd6420e66ca63f44177bc78df89f912" +"checksum tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd8a8b911301c60cbfaa2a6588fb210e5c1038375b8bdecc47aa09a94c3c05f" +"checksum tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3a52f00c97fedb6d535d27f65cccb7181c8dd4c6edc3eda9ea93f6d45d05168e" "checksum tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "da941144b816d0dcda4db3a1ba87596e4df5e860a72b70783fe435891f80601c" -"checksum tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "424c1ed15a0132251813ccea50640b224c809d6ceafb88154c1a8775873a0e89" +"checksum tokio-uds 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "22e3aa6d1fcc19e635418dc0a30ab5bd65d347973d6f43f1a37bf8d9d1335fc9" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum tungstenite 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91541638322b14335fa96c3a4167d9424de72f22986b9d12a52350873b4e83f1" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" @@ -1492,7 +1469,7 @@ dependencies = [ "checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" -"checksum warp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0eb4aa84ff413b1f030e5afd9338c80509df688cf113fab507da7d7fa8979827" +"checksum warp 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f3c669972fab8d3d249cb4516512448b0ae143433e23eac1a157081e8752f5a9" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/Cargo.toml b/Cargo.toml index 6c69591b9..d778ce63b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,36 @@ -[workspace] -members = [ - "raptor", - "raptor-indexer", - "raptor-indexer-csv", - "raptor-search", - "raptor-http", -] +[package] +edition = "2018" +name = "raptor-cli" +version = "0.1.0" +authors = ["Clément Renault "] + +[dependencies] +raptor = { path = "raptor" } +rocksdb = { git = "https://github.com/pingcap/rust-rocksdb.git" } +serde_derive = "1.0" +structopt = "0.2" +serde = "1.0" +warp = { version = "0.1", optional = true } +serde_json = { version = "1.0", optional = true } +csv = { version = "1.0", optional = true } +moby-name-gen = { version = "0.1", optional = true } +unidecode = { version = "0.3", optional = true } +elapsed = { version = "0.1", optional = true } + +[dependencies.fst] +git = "https://github.com/Kerollmops/fst.git" +branch = "automaton-for-deref" + +[features] +default = ["index-csv", "serve-http", "serve-console"] + +index-jsonlines = ["index", "serde_json"] +index-csv = ["index", "csv"] +index = ["moby-name-gen", "unidecode"] + +serve-http = ["serve", "warp", "serde_json"] +serve-console = ["serve", "elapsed"] +serve = [] [profile.release] lto = true diff --git a/raptor-http/Cargo.toml b/raptor-http/Cargo.toml deleted file mode 100644 index 017731ede..000000000 --- a/raptor-http/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -edition = "2018" -name = "raptor-http" -version = "0.1.0" -authors = ["Clément Renault "] - -[dependencies] -raptor = { path = "../raptor" } -serde_derive = "1.0" -serde_json = "1.0" -structopt = "0.2" -serde = "1.0" -warp = "0.1" - -[dependencies.fst] -git = "https://github.com/Kerollmops/fst.git" -branch = "automaton-for-deref" - -[dependencies.rocksdb] -git = "https://github.com/pingcap/rust-rocksdb.git" diff --git a/raptor-http/src/main.rs b/raptor-http/src/main.rs deleted file mode 100644 index 9e81f36d5..000000000 --- a/raptor-http/src/main.rs +++ /dev/null @@ -1,137 +0,0 @@ -#[macro_use] extern crate serde_derive; - -use std::fs::File; -use std::net::SocketAddr; -use structopt::StructOpt; -use std::path::{Path, PathBuf}; -use std::collections::hash_set::HashSet; -use std::io::{self, BufReader, BufRead, Write}; -use std::sync::Arc; -use std::error::Error; -use std::str::from_utf8_unchecked; -use rocksdb::{DB, DBOptions, IngestExternalFileOptions}; -use raptor::{automaton, Metadata, RankedStream}; -use fst::Streamer; -use warp::Filter; - -#[derive(Debug, StructOpt)] -#[structopt(name = "raptor-http", about = "Raptor but wrapped by an http server.")] -struct Opt { - /// The addresse and port to bind the server to. - #[structopt(short = "l", default_value = "127.0.0.1:3030")] - listen_addr: SocketAddr, - - /// The stop word file, each word must be separated by a newline. - #[structopt(long = "stop-words", parse(from_os_str))] - stop_words: PathBuf, - - /// Meta file name (e.g. relaxed-colden). - meta_name: String, -} - -#[derive(Debug, Deserialize)] -struct SearchQuery { q: String } - -#[derive(Debug, Serialize)] -struct Document<'a> { - id: u64, - title: &'a str, - description: &'a str, - image: &'a str, -} - -type CommonWords = HashSet; - -fn common_words

(path: P) -> io::Result -where P: AsRef, -{ - let file = File::open(path)?; - let file = BufReader::new(file); - let mut set = HashSet::new(); - for line in file.lines().filter_map(|l| l.ok()) { - for word in line.split_whitespace() { - set.insert(word.to_owned()); - } - } - Ok(set) -} - -fn search(metadata: M, database: D, common_words: &CommonWords, query: &str) -> Result> -where M: AsRef, - D: AsRef, -{ - let mut automatons = Vec::new(); - for query in query.split_whitespace().map(str::to_lowercase) { - if common_words.contains(&query) { continue } - let lev = automaton::build(&query); - automatons.push(lev); - } - - let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20); - let mut body = Vec::new(); - write!(&mut body, "[")?; - - let mut first = true; - while let Some(document) = stream.next() { - let title_key = format!("{}-title", document.document_id); - let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap(); - let title = unsafe { from_utf8_unchecked(&title) }; - - let description_key = format!("{}-description", document.document_id); - let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap(); - let description = unsafe { from_utf8_unchecked(&description) }; - - let image_key = format!("{}-image", document.document_id); - let image = database.as_ref().get(image_key.as_bytes()).unwrap().unwrap(); - let image = unsafe { from_utf8_unchecked(&image) }; - - let document = Document { - id: document.document_id, - title: title, - description: description, - image: image, - }; - - if !first { write!(&mut body, ",")? } - serde_json::to_writer(&mut body, &document)?; - - first = false; - } - - write!(&mut body, "]")?; - - Ok(String::from_utf8(body)?) -} - -fn main() { - let opt = Opt::from_args(); - - let name = opt.meta_name; - let map_file = format!("{}.map", name); - let idx_file = format!("{}.idx", name); - let sst_file = format!("{}.sst", name); - - let rocksdb = "rocksdb/storage"; - - let meta = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() }; - let meta = Arc::new(meta); - - let db = DB::open_default(rocksdb).unwrap(); - db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap(); - drop(db); - let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap(); - let db = Arc::new(db); - - let common_words = common_words(opt.stop_words).expect("reading stop words"); - - let routes = warp::path("search") - .and(warp::query()) - .map(move |query: SearchQuery| { - let body = search(meta.clone(), db.clone(), &common_words, &query.q).unwrap(); - body - }) - .with(warp::reply::with::header("Content-Type", "application/json")) - .with(warp::reply::with::header("Access-Control-Allow-Origin", "*")); - - warp::serve(routes).run(opt.listen_addr); -} diff --git a/raptor-indexer-csv/.gitignore b/raptor-indexer-csv/.gitignore deleted file mode 100644 index 70e3cae73..000000000 --- a/raptor-indexer-csv/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -/target -**/*.rs.bk diff --git a/raptor-indexer-csv/Cargo.toml b/raptor-indexer-csv/Cargo.toml deleted file mode 100644 index 34a56316f..000000000 --- a/raptor-indexer-csv/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -edition = "2018" -name = "raptor-indexer-csv" -version = "0.1.0" -authors = ["Kerollmops "] - -[dependencies] -raptor = { path = "../raptor" } -serde = "1.0" -structopt = "0.2" -serde_derive = "1.0" -csv = "1.0" -unidecode = "0.3" -moby-name-gen = "0.1" - -[dependencies.rocksdb] -git = "https://github.com/pingcap/rust-rocksdb.git" diff --git a/raptor-indexer-csv/src/main.rs b/raptor-indexer-csv/src/main.rs deleted file mode 100644 index 763a4c962..000000000 --- a/raptor-indexer-csv/src/main.rs +++ /dev/null @@ -1,143 +0,0 @@ -// TODO make the raptor binary expose multiple subcommand -// make only one binary - -#[macro_use] extern crate serde_derive; - -use std::path::{Path, PathBuf}; -use std::collections::{HashSet, BTreeMap}; -use std::io::{self, BufReader, BufRead}; -use std::fs::File; - -use csv::ReaderBuilder; -use structopt::StructOpt; -use raptor::{MetadataBuilder, DocIndex, Tokenizer}; -use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions}; -use unidecode::unidecode; - -#[derive(Debug, StructOpt)] -#[structopt(name = "raptor-indexer-csv", about = "A Raptor binary to index csv stored products.")] -struct Opt { - /// The stop word file, each word must be separated by a newline. - #[structopt(long = "stop-words", parse(from_os_str))] - stop_words: PathBuf, - - /// The csv file to index. - #[structopt(parse(from_os_str))] - products: PathBuf, -} - -#[derive(Debug, Deserialize)] -struct Product { - #[serde(rename = "_unit_id")] - id: u64, - #[serde(rename = "product_title")] - title: String, - #[serde(rename = "product_image")] - image: String, - #[serde(rename = "product_description")] - description: String, -} - -type CommonWords = HashSet; - -fn common_words

(path: P) -> io::Result -where P: AsRef, -{ - let file = File::open(path)?; - let file = BufReader::new(file); - let mut set = HashSet::new(); - for line in file.lines().filter_map(|l| l.ok()) { - for word in line.split_whitespace() { - set.insert(word.to_owned()); - } - } - Ok(set) -} - -fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder, doc_index: u64, attr: u8, words: I) -where A: io::Write, - B: io::Write, - I: IntoIterator, -{ - for (index, word) in words { - let doc_index = DocIndex { - document: doc_index, - attribute: attr, - attribute_index: index as u32, - }; - // insert the exact representation - let word_lower = word.to_lowercase(); - - // and the unidecoded lowercased version - let word_unidecoded = unidecode(word).to_lowercase(); - if word_lower != word_unidecoded { - builder.insert(word_unidecoded, doc_index); - } - - builder.insert(word_lower, doc_index); - } -} - -fn main() { - let opt = Opt::from_args(); - - let common_words = common_words(opt.stop_words).expect("reading stop words"); - - // TODO add a subcommand to pack these files in a tar.xxx archive - let random_name = moby_name_gen::random_name(); - let map_file = format!("{}.map", random_name); - let idx_file = format!("{}.idx", random_name); - let sst_file = format!("{}.sst", random_name); - - let env_options = EnvOptions::new(); - let cf_options = ColumnFamilyOptions::new(); - let mut sst_file_writer = SstFileWriter::new(env_options, cf_options); - sst_file_writer.open(&sst_file).expect("open the sst file"); - - let map = File::create(&map_file).unwrap(); - let indexes = File::create(&idx_file).unwrap(); - let mut builder = MetadataBuilder::new(map, indexes); - let mut fields = BTreeMap::new(); - - let mut rdr = ReaderBuilder::new().from_path(opt.products).expect("reading product file"); - let mut errors = 0; - - for result in rdr.deserialize() { - let product: Product = match result { - Ok(product) => product, - Err(e) => { eprintln!("{:?}", e); errors += 1; continue }, - }; - - let title = Tokenizer::new(&product.title); - let title = title.iter().filter(|&(_, w)| !common_words.contains(w)); - insert_document_words(&mut builder, product.id, 0, title); - - let description = Tokenizer::new(&product.description); - let description = description.iter().filter(|&(_, w)| !common_words.contains(w)); - insert_document_words(&mut builder, product.id, 1, description); - - // TODO simplify this by using functions and - // use the MetadataBuilder internal BTreeMap ? - let key = format!("{}-title", product.id); - let value = product.title; - fields.insert(key, value); - - let key = format!("{}-description", product.id); - let value = product.description; - fields.insert(key, value); - - let key = format!("{}-image", product.id); - let value = product.image; - fields.insert(key, value); - } - - for (key, value) in fields { - sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap(); - } - let _sst_file_info = sst_file_writer.finish().unwrap(); - - builder.finish().unwrap(); - - println!("Found {} errorneous lines", errors); - println!("Succesfully created {:?} dump.", random_name); -} diff --git a/raptor-indexer/.gitignore b/raptor-indexer/.gitignore deleted file mode 100644 index 70e3cae73..000000000 --- a/raptor-indexer/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -/target -**/*.rs.bk diff --git a/raptor-indexer/Cargo.toml b/raptor-indexer/Cargo.toml deleted file mode 100644 index eed5086b6..000000000 --- a/raptor-indexer/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -edition = "2018" -name = "raptor-indexer" -version = "0.1.0" -authors = ["Kerollmops "] - -[dependencies] -raptor = { path = "../raptor" } -serde = "1.0" -serde_derive = "1.0" -serde_json = "1.0" -unidecode = "0.3" -moby-name-gen = "0.1" - -[dependencies.rocksdb] -git = "https://github.com/pingcap/rust-rocksdb.git" diff --git a/raptor-indexer/src/main.rs b/raptor-indexer/src/main.rs deleted file mode 100644 index 78c37450e..000000000 --- a/raptor-indexer/src/main.rs +++ /dev/null @@ -1,123 +0,0 @@ -// TODO make the raptor binary expose multiple subcommand -// make only one binary - -#[macro_use] extern crate serde_derive; - -use std::path::Path; -use std::collections::{HashSet, BTreeMap}; -use std::io::{self, BufReader, BufRead}; -use std::fs::File; -use std::env; - -use raptor::{MetadataBuilder, DocIndex, Tokenizer}; -use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions}; -use serde_json::from_str; -use unidecode::unidecode; - -#[derive(Debug, Deserialize)] -struct Product { - title: String, - group_id: u64, - ft: String, -} - -type CommonWords = HashSet; - -fn common_words

(path: P) -> io::Result -where P: AsRef, -{ - let file = File::open(path)?; - let file = BufReader::new(file); - let mut set = HashSet::new(); - for line in file.lines().filter_map(|l| l.ok()) { - for word in line.split_whitespace() { - set.insert(word.to_owned()); - } - } - Ok(set) -} - -fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder, doc_index: u64, attr: u8, words: I) -where A: io::Write, - B: io::Write, - I: IntoIterator, -{ - for (index, word) in words { - let doc_index = DocIndex { - document: doc_index, - attribute: attr, - attribute_index: index as u32, - }; - // insert the exact representation - let word_lower = word.to_lowercase(); - - // and the unidecoded lowercased version - let word_unidecoded = unidecode(word).to_lowercase(); - if word_lower != word_unidecoded { - builder.insert(word_unidecoded, doc_index); - } - - builder.insert(word_lower, doc_index); - } -} - -fn main() { - let data_path = env::args().nth(1).expect("Missing data json lines file (e.g. products.json_lines)"); - let data = File::open(data_path).unwrap(); - let data = BufReader::new(data); - - let common_path = "fr.stopwords.txt"; - let common_words = common_words(common_path).unwrap_or_else(|e| { - println!("{:?}: {:?}", common_path, e); - HashSet::new() - }); - - // TODO add a subcommand to pack these files in a tar.xxx archive - let random_name = moby_name_gen::random_name(); - let map_file = format!("{}.map", random_name); - let idx_file = format!("{}.idx", random_name); - let sst_file = format!("{}.sst", random_name); - - let env_options = EnvOptions::new(); - let cf_options = ColumnFamilyOptions::new(); - let mut sst_file_writer = SstFileWriter::new(env_options, cf_options); - sst_file_writer.open(&sst_file).expect("open the sst file"); - - let map = File::create(&map_file).unwrap(); - let indexes = File::create(&idx_file).unwrap(); - let mut builder = MetadataBuilder::new(map, indexes); - let mut fields = BTreeMap::new(); - - for line in data.lines() { - let line = line.unwrap(); - - let product: Product = from_str(&line).unwrap(); - - let title = Tokenizer::new(&product.title); - let title = title.iter().filter(|&(_, w)| !common_words.contains(w)); - insert_document_words(&mut builder, product.group_id, 0, title); - - let description = Tokenizer::new(&product.ft); - let description = description.iter().filter(|&(_, w)| !common_words.contains(w)); - insert_document_words(&mut builder, product.group_id, 1, description); - - // TODO simplify this by using functions and - // use the MetadataBuilder internal BTreeMap ? - let key = format!("{}-title", product.group_id); - let value = product.title; - fields.insert(key, value); - - let key = format!("{}-description", product.group_id); - let value = product.ft; - fields.insert(key, value); - } - - for (key, value) in fields { - sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap(); - } - let _sst_file_info = sst_file_writer.finish().unwrap(); - - builder.finish().unwrap(); - - println!("Succesfully created {:?} dump.", random_name); -} diff --git a/raptor-search/.gitignore b/raptor-search/.gitignore deleted file mode 100644 index 70e3cae73..000000000 --- a/raptor-search/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -/target -**/*.rs.bk diff --git a/raptor-search/Cargo.toml b/raptor-search/Cargo.toml deleted file mode 100644 index e9eaf136d..000000000 --- a/raptor-search/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -edition = "2018" -name = "raptor-search" -version = "0.1.0" -authors = ["Kerollmops "] - -[dependencies] -raptor = { path = "../raptor" } -structopt = "0.2" -elapsed = "0.1" - -[dependencies.fst] -git = "https://github.com/Kerollmops/fst.git" -branch = "automaton-for-deref" - -[dependencies.rocksdb] -git = "https://github.com/pingcap/rust-rocksdb.git" diff --git a/raptor-search/benches/main.rs b/raptor-search/benches/main.rs deleted file mode 100644 index f6a962e13..000000000 --- a/raptor-search/benches/main.rs +++ /dev/null @@ -1,29 +0,0 @@ -#![feature(test)] - -extern crate test; -extern crate fst; -extern crate raptor; - -use fst::Streamer; -use raptor::{load_map, RankedStream, LevBuilder}; - -#[bench] -fn chauve_souris(b: &mut test::Bencher) { - let lev_builder = LevBuilder::new(); - let map = load_map("map.fst", "values.vecs").unwrap(); - - let query = "chauve souris"; - - b.iter(|| { - let mut automatons = Vec::new(); - for query in query.split_whitespace() { - let lev = lev_builder.build_automaton(query); - automatons.push(lev); - } - - let mut stream = RankedStream::new(&map, &map.values(), automatons, 20); - while let Some(document_id) = stream.next() { - test::black_box(document_id); - } - }) -} diff --git a/raptor-search/src/main.rs b/raptor-search/src/main.rs deleted file mode 100644 index f3f0891a5..000000000 --- a/raptor-search/src/main.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::fs::File; -use std::path::{Path, PathBuf}; -use std::collections::HashSet; -use std::str::from_utf8_unchecked; -use std::io::{self, BufReader, BufRead, Write}; -use structopt::StructOpt; -use elapsed::measure_time; -use fst::Streamer; -use rocksdb::{DB, DBOptions, IngestExternalFileOptions}; -use raptor::{automaton, Metadata, RankedStream}; - -#[derive(Debug, StructOpt)] -#[structopt(name = "raptor-search", about = "A Raptor binary to search in a dump.")] -struct Opt { - /// The stop word file, each word must be separated by a newline. - #[structopt(long = "stop-words", parse(from_os_str))] - stop_words: PathBuf, - - /// Meta file name (e.g. relaxed-colden). - meta_name: String, -} - -type CommonWords = HashSet; - -fn common_words

(path: P) -> io::Result -where P: AsRef, -{ - let file = File::open(path)?; - let file = BufReader::new(file); - let mut set = HashSet::new(); - for line in file.lines().filter_map(|l| l.ok()) { - for word in line.split_whitespace() { - set.insert(word.to_owned()); - } - } - Ok(set) -} - -fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) { - let mut automatons = Vec::new(); - for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) { - let lev = automaton::build(query); - automatons.push(lev); - } - - let mut stream = RankedStream::new(&metadata, automatons, 20); - while let Some(document) = stream.next() { - print!("{:?}", document.document_id); - - let title_key = format!("{}-title", document.document_id); - let title = database.get(title_key.as_bytes()).unwrap().unwrap(); - let title = unsafe { from_utf8_unchecked(&title) }; - print!(" {:?}", title); - - println!(); - } -} - -fn main() { - let opt = Opt::from_args(); - - let name = opt.meta_name; - let map_file = format!("{}.map", name); - let idx_file = format!("{}.idx", name); - let sst_file = format!("{}.sst", name); - - let rocksdb = "rocksdb/storage"; - - let (elapsed, meta) = measure_time(|| unsafe { - Metadata::from_paths(map_file, idx_file).unwrap() - }); - println!("{} to load metadata", elapsed); - - let (elapsed, db) = measure_time(|| { - let db = DB::open_default(rocksdb).unwrap(); - db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap(); - drop(db); - DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap() - }); - println!("{} to load the SST file in RocksDB and reopen it for read-only", elapsed); - - let common_words = common_words(opt.stop_words).expect("reading stop words"); - - loop { - print!("Searching for: "); - io::stdout().flush().unwrap(); - - let mut query = String::new(); - io::stdin().read_line(&mut query).unwrap(); - let query = query.trim().to_lowercase(); - - if query.is_empty() { break } - - let (elapsed, _) = measure_time(|| search(&meta, &db, &common_words, &query)); - println!("Finished in {}", elapsed); - } -} diff --git a/src/common_words.rs b/src/common_words.rs new file mode 100644 index 000000000..2fc5f5259 --- /dev/null +++ b/src/common_words.rs @@ -0,0 +1,20 @@ +use std::io::{self, BufReader, BufRead}; +use std::collections::HashSet; +use std::path::Path; +use std::fs::File; + +pub type CommonWords = HashSet; + +pub fn from_file

(path: P) -> io::Result +where P: AsRef, +{ + let file = File::open(path)?; + let file = BufReader::new(file); + let mut set = HashSet::new(); + for line in file.lines().filter_map(|l| l.ok()) { + for word in line.split_whitespace() { + set.insert(word.to_owned()); + } + } + Ok(set) +} diff --git a/src/index/csv.rs b/src/index/csv.rs new file mode 100644 index 000000000..7accefeec --- /dev/null +++ b/src/index/csv.rs @@ -0,0 +1,122 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::fs::File; +use std::io; + +use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions}; +use raptor::{MetadataBuilder, DocIndex, Tokenizer}; +use unidecode::unidecode; +use csv::ReaderBuilder; + +use crate::common_words::{self, CommonWords}; +use crate::index::csv_feature::CommandCsv; + +#[derive(Debug, Deserialize)] +struct Product { + #[serde(rename = "_unit_id")] + id: u64, + #[serde(rename = "product_title")] + title: String, + #[serde(rename = "product_image")] + image: String, + #[serde(rename = "product_description")] + description: String, +} + +#[derive(Debug)] +pub struct CsvIndexer { + common_words: CommonWords, + products: PathBuf, +} + +impl CsvIndexer { + pub fn from_command(command: CommandCsv) -> io::Result { + let common_words = common_words::from_file(command.stop_words)?; + let products = command.products; + + Ok(CsvIndexer { common_words, products }) + } + + pub fn index(self) { + let random_name = moby_name_gen::random_name(); + let map_file = format!("{}.map", random_name); + let idx_file = format!("{}.idx", random_name); + let sst_file = format!("{}.sst", random_name); + + let env_options = EnvOptions::new(); + let cf_options = ColumnFamilyOptions::new(); + let mut sst_file_writer = SstFileWriter::new(env_options, cf_options); + sst_file_writer.open(&sst_file).expect("open the sst file"); + + let map = File::create(&map_file).unwrap(); + let indexes = File::create(&idx_file).unwrap(); + let mut builder = MetadataBuilder::new(map, indexes); + let mut fields = BTreeMap::new(); + + let mut rdr = ReaderBuilder::new().from_path(&self.products).expect("reading product file"); + let mut errors = 0; + + for result in rdr.deserialize() { + let product: Product = match result { + Ok(product) => product, + Err(e) => { eprintln!("{:?}", e); errors += 1; continue }, + }; + + let title = Tokenizer::new(&product.title); + let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w)); + insert_document_words(&mut builder, product.id, 0, title); + + let description = Tokenizer::new(&product.description); + let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w)); + insert_document_words(&mut builder, product.id, 1, description); + + // TODO simplify this by using functions and + // use the MetadataBuilder internal BTreeMap ? + let key = format!("{}-title", product.id); + let value = product.title; + fields.insert(key, value); + + let key = format!("{}-description", product.id); + let value = product.description; + fields.insert(key, value); + + let key = format!("{}-image", product.id); + let value = product.image; + fields.insert(key, value); + } + + for (key, value) in fields { + sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap(); + } + let _sst_file_info = sst_file_writer.finish().unwrap(); + + builder.finish().unwrap(); + + println!("Found {} errorneous lines", errors); + println!("Succesfully created {:?} dump.", random_name); + } +} + +fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder, doc_index: u64, attr: u8, words: I) +where A: io::Write, + B: io::Write, + I: IntoIterator, +{ + for (index, word) in words { + let doc_index = DocIndex { + document: doc_index, + attribute: attr, + attribute_index: index as u32, + }; + // insert the exact representation + let word_lower = word.to_lowercase(); + + // and the unidecoded lowercased version + let word_unidecoded = unidecode(word).to_lowercase(); + if word_lower != word_unidecoded { + builder.insert(word_unidecoded, doc_index); + } + + builder.insert(word_lower, doc_index); + } +} diff --git a/src/index/json_lines.rs b/src/index/json_lines.rs new file mode 100644 index 000000000..6c2d2636b --- /dev/null +++ b/src/index/json_lines.rs @@ -0,0 +1,112 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::fs::File; +use std::io::{self, BufReader, BufRead}; + +use serde_json::from_str; +use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions}; +use raptor::{MetadataBuilder, DocIndex, Tokenizer}; +use unidecode::unidecode; + +use crate::common_words::{self, CommonWords}; +use crate::index::jsonlines_feature::CommandJsonLines; + +#[derive(Debug, Deserialize)] +struct Product { + title: String, + group_id: u64, + ft: String, +} + +#[derive(Debug)] +pub struct JsonLinesIndexer { + common_words: CommonWords, + products: PathBuf, +} + +impl JsonLinesIndexer { + pub fn from_command(command: CommandJsonLines) -> io::Result { + let common_words = common_words::from_file(command.stop_words)?; + let products = command.products; + + Ok(JsonLinesIndexer { common_words, products }) + } + + pub fn index(self) { + let data = File::open(&self.products).unwrap(); + let data = BufReader::new(data); + + // TODO add a subcommand to pack these files in a tar.xxx archive + let random_name = moby_name_gen::random_name(); + let map_file = format!("{}.map", random_name); + let idx_file = format!("{}.idx", random_name); + let sst_file = format!("{}.sst", random_name); + + let env_options = EnvOptions::new(); + let cf_options = ColumnFamilyOptions::new(); + let mut sst_file_writer = SstFileWriter::new(env_options, cf_options); + sst_file_writer.open(&sst_file).expect("open the sst file"); + + let map = File::create(&map_file).unwrap(); + let indexes = File::create(&idx_file).unwrap(); + let mut builder = MetadataBuilder::new(map, indexes); + let mut fields = BTreeMap::new(); + + for line in data.lines() { + let line = line.unwrap(); + + let product: Product = from_str(&line).unwrap(); + + let title = Tokenizer::new(&product.title); + let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w)); + insert_document_words(&mut builder, product.group_id, 0, title); + + let description = Tokenizer::new(&product.ft); + let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w)); + insert_document_words(&mut builder, product.group_id, 1, description); + + // TODO simplify this by using functions and + // use the MetadataBuilder internal BTreeMap ? + let key = format!("{}-title", product.group_id); + let value = product.title; + fields.insert(key, value); + + let key = format!("{}-description", product.group_id); + let value = product.ft; + fields.insert(key, value); + } + + for (key, value) in fields { + sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap(); + } + let _sst_file_info = sst_file_writer.finish().unwrap(); + + builder.finish().unwrap(); + + println!("Succesfully created {:?} dump.", random_name); + } +} + +fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder, doc_index: u64, attr: u8, words: I) +where A: io::Write, + B: io::Write, + I: IntoIterator, +{ + for (index, word) in words { + let doc_index = DocIndex { + document: doc_index, + attribute: attr, + attribute_index: index as u32, + }; + // insert the exact representation + let word_lower = word.to_lowercase(); + + // and the unidecoded lowercased version + let word_unidecoded = unidecode(word).to_lowercase(); + if word_lower != word_unidecoded { + builder.insert(word_unidecoded, doc_index); + } + + builder.insert(word_lower, doc_index); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs new file mode 100644 index 000000000..9966afe3c --- /dev/null +++ b/src/index/mod.rs @@ -0,0 +1,71 @@ +#[cfg(feature = "index-csv")] +mod csv; + +#[cfg(feature = "index-jsonlines")] +mod json_lines; + +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +pub enum CommandIndex { + + #[cfg(feature = "index-jsonlines")] + /// Index files encoded as json lines. + #[structopt(name = "json-lines")] + JsonLines(self::jsonlines_feature::CommandJsonLines), + + #[cfg(feature = "index-csv")] + /// Index files encoded as csv. + #[structopt(name = "csv")] + Csv(self::csv_feature::CommandCsv), +} + +#[cfg(feature = "index-jsonlines")] +pub mod jsonlines_feature { + use std::error; + use std::path::PathBuf; + use structopt::StructOpt; + + #[derive(Debug, StructOpt)] + pub struct CommandJsonLines { + /// The stop word file, each word must be separated by a newline. + #[structopt(long = "stop-words", parse(from_os_str))] + pub stop_words: PathBuf, + + /// The csv file to index. + #[structopt(parse(from_os_str))] + pub products: PathBuf, + } + + pub fn json_lines(command: CommandJsonLines) -> Result<(), Box> { + use super::json_lines::JsonLinesIndexer; + + let indexer = JsonLinesIndexer::from_command(command)?; + Ok(indexer.index()) + } +} + +#[cfg(feature = "index-csv")] +pub mod csv_feature { + use std::error; + use std::path::PathBuf; + use structopt::StructOpt; + + #[derive(Debug, StructOpt)] + pub struct CommandCsv { + /// The stop word file, each word must be separated by a newline. + #[structopt(long = "stop-words", parse(from_os_str))] + pub stop_words: PathBuf, + + /// The csv file to index. + #[structopt(parse(from_os_str))] + pub products: PathBuf, + } + + pub fn csv(command: CommandCsv) -> Result<(), Box> { + use super::csv::CsvIndexer; + + let indexer = CsvIndexer::from_command(command)?; + Ok(indexer.index()) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 000000000..ff1bb146d --- /dev/null +++ b/src/main.rs @@ -0,0 +1,50 @@ +#[macro_use] extern crate serde_derive; + +#[cfg(feature = "index")] +mod index; +#[cfg(feature = "serve")] +mod serve; +mod common_words; + +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "raptor-cli", about = "A command line to do raptor operations.")] +enum Commands { + #[cfg(feature = "index")] + /// Index files of different format. + #[structopt(name = "index")] + Index(index::CommandIndex), + + #[cfg(feature = "serve")] + /// Serve indexes. + #[structopt(name = "serve")] + Serve(serve::CommandServe), +} + +fn main() { + let ret = match Commands::from_args() { + + #[cfg(feature = "index")] + Commands::Index(i) => match i { + + #[cfg(feature = "index-jsonlines")] + index::CommandIndex::JsonLines(command) => index::jsonlines_feature::json_lines(command), + + #[cfg(feature = "index-csv")] + index::CommandIndex::Csv(command) => index::csv_feature::csv(command), + }, + + #[cfg(feature = "serve")] + Commands::Serve(s) => match s { + + #[cfg(feature = "serve-http")] + serve::CommandServe::Http(command) => serve::http_feature::http(command), + + #[cfg(feature = "serve-console")] + serve::CommandServe::Console(command) => serve::console_feature::console(command), + }, + }; + + if let Err(e) = ret { eprintln!("{}", e) } +} diff --git a/src/serve/console.rs b/src/serve/console.rs new file mode 100644 index 000000000..8a2f6489f --- /dev/null +++ b/src/serve/console.rs @@ -0,0 +1,72 @@ +use std::str::from_utf8_unchecked; +use std::io::{self, Write}; + +use fst::Streamer; +use elapsed::measure_time; +use rocksdb::{DB, DBOptions, IngestExternalFileOptions}; +use raptor::{automaton, Metadata, RankedStream}; + +use crate::serve::console_feature::CommandConsole; +use crate::common_words::{self, CommonWords}; + +pub struct ConsoleSearch { + common_words: CommonWords, + metadata: Metadata, + db: DB, +} + +impl ConsoleSearch { + pub fn from_command(command: CommandConsole) -> io::Result { + let common_words = common_words::from_file(command.stop_words)?; + + let meta_name = command.meta_name.display(); + let map_file = format!("{}.map", meta_name); + let idx_file = format!("{}.idx", meta_name); + let sst_file = format!("{}.sst", meta_name); + let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() }; + + let rocksdb = "rocksdb/storage"; + let db = DB::open_default(rocksdb).unwrap(); + db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap(); + drop(db); + let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap(); + + Ok(ConsoleSearch { common_words, metadata, db }) + } + + pub fn serve(self) { + loop { + print!("Searching for: "); + io::stdout().flush().unwrap(); + + let mut query = String::new(); + io::stdin().read_line(&mut query).unwrap(); + let query = query.trim().to_lowercase(); + + if query.is_empty() { break } + + let (elapsed, _) = measure_time(|| search(&self.metadata, &self.db, &self.common_words, &query)); + println!("Finished in {}", elapsed); + } + } +} + +fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) { + let mut automatons = Vec::new(); + for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) { + let lev = automaton::build(query); + automatons.push(lev); + } + + let mut stream = RankedStream::new(&metadata, automatons, 20); + while let Some(document) = stream.next() { + print!("{:?}", document.document_id); + + let title_key = format!("{}-title", document.document_id); + let title = database.get(title_key.as_bytes()).unwrap().unwrap(); + let title = unsafe { from_utf8_unchecked(&title) }; + print!(" {:?}", title); + + println!(); + } +} diff --git a/src/serve/http.rs b/src/serve/http.rs new file mode 100644 index 000000000..2ec63050a --- /dev/null +++ b/src/serve/http.rs @@ -0,0 +1,120 @@ +use std::str::from_utf8_unchecked; +use std::io::{self, Write}; +use std::net::SocketAddr; +use std::error::Error; +use std::sync::Arc; + +use rocksdb::{DB, DBOptions, IngestExternalFileOptions}; +use raptor::{automaton, Metadata}; +use raptor::rank::RankedStream; +use fst::Streamer; +use warp::Filter; + +use crate::serve::http_feature::CommandHttp; +use crate::common_words::{self, CommonWords}; + +#[derive(Debug, Serialize)] +struct Document<'a> { + id: u64, + title: &'a str, + description: &'a str, + image: &'a str, +} + +pub struct HttpServer { + listen_addr: SocketAddr, + common_words: Arc, + metadata: Arc, + db: Arc, +} + +#[derive(Debug, Deserialize)] +struct SearchQuery { q: String } + +impl HttpServer { + pub fn from_command(command: CommandHttp) -> io::Result { + let common_words = common_words::from_file(command.stop_words)?; + + let meta_name = command.meta_name.display(); + let map_file = format!("{}.map", meta_name); + let idx_file = format!("{}.idx", meta_name); + let sst_file = format!("{}.sst", meta_name); + let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() }; + + let rocksdb = "rocksdb/storage"; + let db = DB::open_default(rocksdb).unwrap(); + db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap(); + drop(db); + let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap(); + + Ok(HttpServer { + listen_addr: command.listen_addr, + common_words: Arc::new(common_words), + metadata: Arc::new(metadata), + db: Arc::new(db), + }) + } + + pub fn serve(self) { + let HttpServer { listen_addr, common_words, metadata, db } = self; + + let routes = warp::path("search") + .and(warp::query()) + .map(move |query: SearchQuery| { + let body = search(metadata.clone(), db.clone(), common_words.clone(), &query.q).unwrap(); + body + }) + .with(warp::reply::with::header("Content-Type", "application/json")) + .with(warp::reply::with::header("Access-Control-Allow-Origin", "*")); + + warp::serve(routes).run(listen_addr) + } +} + +fn search(metadata: M, database: D, common_words: C, query: &str) -> Result> +where M: AsRef, + D: AsRef, + C: AsRef, +{ + let mut automatons = Vec::new(); + for query in query.split_whitespace().map(str::to_lowercase) { + if common_words.as_ref().contains(&query) { continue } + let lev = automaton::build(&query); + automatons.push(lev); + } + + let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20); + let mut body = Vec::new(); + write!(&mut body, "[")?; + + let mut first = true; + while let Some(document) = stream.next() { + let title_key = format!("{}-title", document.document_id); + let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap(); + let title = unsafe { from_utf8_unchecked(&title) }; + + let description_key = format!("{}-description", document.document_id); + let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap(); + let description = unsafe { from_utf8_unchecked(&description) }; + + let image_key = format!("{}-image", document.document_id); + let image = database.as_ref().get(image_key.as_bytes()).unwrap().unwrap(); + let image = unsafe { from_utf8_unchecked(&image) }; + + let document = Document { + id: document.document_id, + title: title, + description: description, + image: image, + }; + + if !first { write!(&mut body, ",")? } + serde_json::to_writer(&mut body, &document)?; + + first = false; + } + + write!(&mut body, "]")?; + + Ok(String::from_utf8(body)?) +} diff --git a/src/serve/mod.rs b/src/serve/mod.rs new file mode 100644 index 000000000..447f8471b --- /dev/null +++ b/src/serve/mod.rs @@ -0,0 +1,76 @@ +#[cfg(feature = "serve-http")] +mod http; + +#[cfg(feature = "serve-console")] +mod console; + +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +pub enum CommandServe { + + #[cfg(feature = "serve-http")] + /// Serve an index under an http protocol. + #[structopt(name = "http")] + Http(self::http_feature::CommandHttp), + + #[cfg(feature = "serve-console")] + /// Serve an index under a simple console. + #[structopt(name = "console")] + Console(self::console_feature::CommandConsole), +} + +#[cfg(feature = "serve-http")] +pub mod http_feature { + use std::error; + use std::path::PathBuf; + use std::net::SocketAddr; + use structopt::StructOpt; + + #[derive(Debug, StructOpt)] + pub struct CommandHttp { + /// The address and port to bind the server to. + #[structopt(short = "l", default_value = "127.0.0.1:3030")] + pub listen_addr: SocketAddr, + + /// The stop word file, each word must be separated by a newline. + #[structopt(long = "stop-words", parse(from_os_str))] + pub stop_words: PathBuf, + + /// Meta file name (e.g. relaxed-colden). + #[structopt(parse(from_os_str))] + pub meta_name: PathBuf, + } + + pub fn http(command: CommandHttp) -> Result<(), Box> { + use super::http::HttpServer; + + let server = HttpServer::from_command(command)?; + Ok(server.serve()) + } +} + +#[cfg(feature = "serve-console")] +pub mod console_feature { + use std::error; + use std::path::PathBuf; + use structopt::StructOpt; + + #[derive(Debug, StructOpt)] + pub struct CommandConsole { + /// The stop word file, each word must be separated by a newline. + #[structopt(long = "stop-words", parse(from_os_str))] + pub stop_words: PathBuf, + + /// Meta file name (e.g. relaxed-colden). + #[structopt(parse(from_os_str))] + pub meta_name: PathBuf, + } + + pub fn console(command: CommandConsole) -> Result<(), Box> { + use super::console::ConsoleSearch; + + let search = ConsoleSearch::from_command(command)?; + Ok(search.serve()) + } +}