searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

feat: Make a CLI that handle compile-time features

Browse Source
This commit is contained in:
Clément Renault 2018-10-03 16:21:33 +02:00
parent a066c084fe
commit 474028145d
22 changed files with 811 additions and 774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
Cargo.lock generated
View File

@ -39,11 +39,11 @@ dependencies = [
[[package]]
name = "base64"
version = "0.9.2"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -179,7 +179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "csv"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -196,7 +196,7 @@ dependencies = [
[[package]]
name = "digest"
version = "0.7.5"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -291,52 +291,51 @@ dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "http"
version = "0.1.12"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "httparse"
version = "1.3.2"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hyper"
version = "0.12.9"
version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -374,7 +373,7 @@ dependencies = [
[[package]]
name = "itoa"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -396,7 +395,7 @@ dependencies = [
[[package]]
name = "lazycell"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -440,7 +439,7 @@ dependencies = [
[[package]]
name = "lock_api"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -521,7 +520,7 @@ dependencies = [
"fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazycell 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -595,7 +594,7 @@ name = "parking_lot"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -658,7 +657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "proc-macro2"
version = "0.4.13"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -669,7 +668,7 @@ name = "quote"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -724,55 +723,21 @@ dependencies = [
]
[[package]]
name = "raptor-http"
version = "0.1.0"
dependencies = [
"fst 0.3.2 (git+https://github.com/Kerollmops/fst.git?branch=automaton-for-deref)",
"raptor 0.1.0",
"rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"warp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "raptor-indexer"
version = "0.1.0"
dependencies = [
"moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"raptor 0.1.0",
"rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
"unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "raptor-indexer-csv"
version = "0.1.0"
dependencies = [
"csv 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"raptor 0.1.0",
"rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "raptor-search"
name = "raptor-cli"
version = "0.1.0"
dependencies = [
"csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"fst 0.3.2 (git+https://github.com/Kerollmops/fst.git?branch=automaton-for-deref)",
"moby-name-gen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"raptor 0.1.0",
"rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"warp 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -808,12 +773,12 @@ dependencies = [
[[package]]
name = "ryu"
version = "0.2.5"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "safemem"
version = "0.2.0"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -851,21 +816,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde_derive"
version = "1.0.75"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_json"
version = "1.0.26"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -875,7 +840,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -887,7 +852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
"digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
"fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -948,7 +913,7 @@ name = "structopt-derive"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -958,7 +923,17 @@ name = "syn"
version = "0.14.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "syn"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -993,46 +968,47 @@ dependencies = [
[[package]]
name = "tokio"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-current-thread 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-codec"
version = "0.1.0"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-uds 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-codec"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-current-thread"
version = "0.1.1"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-executor"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1044,13 +1020,13 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-io"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1060,7 +1036,7 @@ dependencies = [
[[package]]
name = "tokio-reactor"
version = "0.1.5"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1071,26 +1047,26 @@ dependencies = [
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-tcp"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-threadpool"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1099,18 +1075,18 @@ dependencies = [
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-timer"
version = "0.2.6"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1122,14 +1098,14 @@ dependencies = [
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tokio-uds"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1139,8 +1115,8 @@ dependencies = [
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
"mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1153,10 +1129,10 @@ name = "tungstenite"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"input_buffer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1274,24 +1250,24 @@ dependencies = [
[[package]]
name = "warp"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.12.9 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
"scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
"tungstenite 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"urlencoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1351,7 +1327,7 @@ dependencies = [
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
"checksum base64 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "30e93c03064e7590d0466209155251b90c22e37fab1daf2771582598b5827557"
"checksum base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "85415d2594767338a74a30c1d370b2f3262ec1b4ed2d7bba5b3faf4de40467d9"
"checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643"
"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
"checksum blob 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "122c3fa3949d822d2a51c648db9e8105d6e75b89dc628cc366901d3d396fa4f4"
"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab"
@ -1369,9 +1345,9 @@ dependencies = [
"checksum crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3486aefc4c0487b9cb52372c97df0a48b8c249514af1ee99703bf70d2f2ceda1"
"checksum crossbeam-epoch 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "30fecfcac6abfef8771151f8be4abc9e4edc112c2bcb233314cafde2680536e9"
"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
"checksum csv 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bbb6d75aae072248e381715437855a69595e5a97d3abbf748fe7a95e65d77fa"
"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452"
"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105"
"checksum digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "5b29c278aa8fd30796bd977169e8004b4aa88cdcd2f32a6eb22bc2d5d38df94a"
"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90"
"checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd"
"checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29"
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
@ -1386,22 +1362,22 @@ dependencies = [
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
"checksum group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)" = "<none>"
"checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c"
"checksum http 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3e5765436510a2021e6917dfb87d1afa4b32d17c71353c013897fa5da29527a1"
"checksum httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b6288d7db100340ca12873fd4d08ad1b8f206a9457798dfb17c018a33fee540"
"checksum hyper 0.12.9 (registry+https://github.com/rust-lang/crates.io-index)" = "081289d17dce471c8cbc0e69a3dd073b627e08338561d1167ab620b754d9fe90"
"checksum http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "24f58e8c2d8e886055c3ead7b28793e1455270b5fb39650984c224bc538ba581"
"checksum httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83"
"checksum hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)" = "78d50abbd1790e0f4c74cb1d4a2211b439bac661d54107ad5564c55e77906762"
"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
"checksum indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08173ba1e906efb6538785a8844dd496f5d34f0a2d88038e95195172fc667220"
"checksum input_buffer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e1b822cc844905551931d6f81608ed5f50a79c1078a4e2b4d42dbc7c1eedfbf"
"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
"checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606"
"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7"
"checksum lazycell 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e26d4c411b39f0afcf2ba6fe502be90e6c9b299c952dbd86124782520a13cffd"
"checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0"
"checksum levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=new-custom-fst)" = "<none>"
"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
"checksum librocksdb_sys 0.1.0 (git+https://github.com/pingcap/rust-rocksdb.git)" = "<none>"
"checksum libz-sys 1.0.18 (git+https://github.com/busyjay/libz-sys.git?branch=static-link)" = "<none>"
"checksum lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "949826a5ccf18c1b3a7c3d57692778d21768b79e46eb9dd07bfc4c2160036c54"
"checksum lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775751a3e69bde4df9b38dd00a1b5d6ac13791e4223d4a0506577f0dd27cfb7a"
"checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f"
"checksum lz4-sys 1.8.0 (git+https://github.com/busyjay/lz4-rs.git?branch=adjust-build)" = "<none>"
"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
@ -1426,7 +1402,7 @@ dependencies = [
"checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b"
"checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93"
"checksum pkg-config 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)" = "104630aa1c83213cbc76db0703630fcb0421dac3585063be4ce9a8a2feeaa745"
"checksum proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "ee5697238f0d893c7f0ecc59c0999f18d2af85e424de441178bcacc9f9e6cf67"
"checksum proc-macro2 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)" = "ffe022fb8c8bd254524b0b3305906c1921fa37a84a644e29079a9e62200c3901"
"checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5"
"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1"
"checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd"
@ -1436,16 +1412,16 @@ dependencies = [
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum rocksdb 0.3.0 (git+https://github.com/pingcap/rust-rocksdb.git)" = "<none>"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c066b8e2923f05d4718a06d2622f189ff362bc642bfade6c6629f0440f3827"
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
"checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7"
"checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9"
"checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28"
"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
"checksum sdset 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d51ad726aa3a9c4d777b35be3a4d6d5f9d6cbc0978e81c7d690d31192f263843"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
"checksum serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)" = "22d340507cea0b7e6632900a176101fea959c7065d93ba555072da90aaaafc87"
"checksum serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)" = "234fc8b737737b148ccd625175fc6390f5e4dacfdaa543cb93a3430d984a9119"
"checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae"
"checksum serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "31569d901045afbff7a9479f793177fe9259819aff10ab4f89ef69bbc5f567fe"
"checksum serde_json 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "bb47a3d5c84320222f66d7db21157c4a7407755de41798f9b4c1c40593397b1a"
"checksum serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aaed41d9fb1e2f587201b863356590c90c1157495d811430a0c0325fe8169650"
"checksum sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "51b9d1f3b5de8a167ab06834a7c883bd197f2191e1dda1a22d9ccfeedbf9aded"
"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
@ -1458,21 +1434,22 @@ dependencies = [
"checksum structopt 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8e9ad6a11096cbecdcca0cc6aa403fdfdbaeda2fb3323a39c98e6a166a1e45a"
"checksum structopt-derive 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4cbce8ccdc62166bd594c14396a3242bf94c337a51dbfa9be1076dd74b3db2af"
"checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741"
"checksum syn 0.15.7 (registry+https://github.com/rust-lang/crates.io-index)" = "455a6ec9b368f8c479b0ae5494d13b22dc00990d2f00d68c9dc6a2dc4f17f210"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b"
"checksum tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fbb6a6e9db2702097bfdfddcb09841211ad423b86c75b5ddaca1d62842ac492c"
"checksum tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "881e9645b81c2ce95fcb799ded2c29ffb9f25ef5bef909089a420e5961dd8ccb"
"checksum tokio-current-thread 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fdfb899688ac16f618076bd09215edbfda0fd5dfecb375b6942636cb31fa8a7"
"checksum tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "84823b932d566bc3c6aa644df4ca36cb38593c50b7db06011fd4e12e31e4047e"
"checksum tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6e93c78d23cc61aa245a8acd2c4a79c4d7fa7fb5c3ca90d5737029f043a84895"
"checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f"
"checksum tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f90fcd90952f0a496d438a976afba8e5c205fb12123f813d8ab3aa1c8436638c"
"checksum tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c117b6cf86bb730aab4834f10df96e4dd586eff2c3c27d3781348da49e255bde"
"checksum tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5cbe4ca6e71cb0b62a66e4e6f53a8c06a6eefe46cc5f665ad6f274c9906f135"
"checksum tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6cc2de7725863c86ac71b0b9068476fec50834f055a243558ef1655bbd34cb"
"checksum tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4bfbaf9f260635649ec26b6fb4aded03887295ffcd999f6e43fd2c4758f758ea"
"checksum tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5b4c329b47f071eb8a746040465fa751bd95e4716e98daef6a9b4e434c17d565"
"checksum tokio-threadpool 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a5758cecb6e0633cea5d563ac07c975e04961690b946b04fd84e7d6445a8f6af"
"checksum tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d03fa701f9578a01b7014f106b47f0a363b4727a7f3f75d666e312ab7acbbf1c"
"checksum tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8a85fffbec3c5ab1ab62324570230dcd37ee5996a7859da5caf7b9d45e3e8c"
"checksum tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4b26fd37f1125738b2170c80b551f69ff6fecb277e6e5ca885e53eec2b005018"
"checksum tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7ad235e9dadd126b2d47f6736f65aa1fdcd6420e66ca63f44177bc78df89f912"
"checksum tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd8a8b911301c60cbfaa2a6588fb210e5c1038375b8bdecc47aa09a94c3c05f"
"checksum tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3a52f00c97fedb6d535d27f65cccb7181c8dd4c6edc3eda9ea93f6d45d05168e"
"checksum tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "da941144b816d0dcda4db3a1ba87596e4df5e860a72b70783fe435891f80601c"
"checksum tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "424c1ed15a0132251813ccea50640b224c809d6ceafb88154c1a8775873a0e89"
"checksum tokio-uds 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "22e3aa6d1fcc19e635418dc0a30ab5bd65d347973d6f43f1a37bf8d9d1335fc9"
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
"checksum tungstenite 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91541638322b14335fa96c3a4167d9424de72f22986b9d12a52350873b4e83f1"
"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
@ -1492,7 +1469,7 @@ dependencies = [
"checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3"
"checksum warp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0eb4aa84ff413b1f030e5afd9338c80509df688cf113fab507da7d7fa8979827"
"checksum warp 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f3c669972fab8d3d249cb4516512448b0ae143433e23eac1a157081e8752f5a9"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"

41
Cargo.toml
View File

@ -1,11 +1,36 @@
[workspace]
members = [
"raptor",
"raptor-indexer",
"raptor-indexer-csv",
"raptor-search",
"raptor-http",
]
[package]
edition = "2018"
name = "raptor-cli"
version = "0.1.0"
authors = ["Clément Renault <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "raptor" }
rocksdb = { git = "https://github.com/pingcap/rust-rocksdb.git" }
serde_derive = "1.0"
structopt = "0.2"
serde = "1.0"
warp = { version = "0.1", optional = true }
serde_json = { version = "1.0", optional = true }
csv = { version = "1.0", optional = true }
moby-name-gen = { version = "0.1", optional = true }
unidecode = { version = "0.3", optional = true }
elapsed = { version = "0.1", optional = true }
[dependencies.fst]
git = "https://github.com/Kerollmops/fst.git"
branch = "automaton-for-deref"
[features]
default = ["index-csv", "serve-http", "serve-console"]
index-jsonlines = ["index", "serde_json"]
index-csv = ["index", "csv"]
index = ["moby-name-gen", "unidecode"]
serve-http = ["serve", "warp", "serde_json"]
serve-console = ["serve", "elapsed"]
serve = []
[profile.release]
lto = true

20
raptor-http/Cargo.toml
View File

@ -1,20 +0,0 @@
[package]
edition = "2018"
name = "raptor-http"
version = "0.1.0"
authors = ["Clément Renault <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "../raptor" }
serde_derive = "1.0"
serde_json = "1.0"
structopt = "0.2"
serde = "1.0"
warp = "0.1"
[dependencies.fst]
git = "https://github.com/Kerollmops/fst.git"
branch = "automaton-for-deref"
[dependencies.rocksdb]
git = "https://github.com/pingcap/rust-rocksdb.git"

137
raptor-http/src/main.rs
View File

@ -1,137 +0,0 @@
#[macro_use] extern crate serde_derive;
use std::fs::File;
use std::net::SocketAddr;
use structopt::StructOpt;
use std::path::{Path, PathBuf};
use std::collections::hash_set::HashSet;
use std::io::{self, BufReader, BufRead, Write};
use std::sync::Arc;
use std::error::Error;
use std::str::from_utf8_unchecked;
use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
use raptor::{automaton, Metadata, RankedStream};
use fst::Streamer;
use warp::Filter;
#[derive(Debug, StructOpt)]
#[structopt(name = "raptor-http", about = "Raptor but wrapped by an http server.")]
struct Opt {
/// The addresse and port to bind the server to.
#[structopt(short = "l", default_value = "127.0.0.1:3030")]
listen_addr: SocketAddr,
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
stop_words: PathBuf,
/// Meta file name (e.g. relaxed-colden).
meta_name: String,
}
#[derive(Debug, Deserialize)]
struct SearchQuery { q: String }
#[derive(Debug, Serialize)]
struct Document<'a> {
id: u64,
title: &'a str,
description: &'a str,
image: &'a str,
}
type CommonWords = HashSet<String>;
fn common_words<P>(path: P) -> io::Result<CommonWords>
where P: AsRef<Path>,
{
let file = File::open(path)?;
let file = BufReader::new(file);
let mut set = HashSet::new();
for line in file.lines().filter_map(|l| l.ok()) {
for word in line.split_whitespace() {
set.insert(word.to_owned());
}
}
Ok(set)
}
fn search<M, D>(metadata: M, database: D, common_words: &CommonWords, query: &str) -> Result<String, Box<Error>>
where M: AsRef<Metadata>,
D: AsRef<DB>,
{
let mut automatons = Vec::new();
for query in query.split_whitespace().map(str::to_lowercase) {
if common_words.contains(&query) { continue }
let lev = automaton::build(&query);
automatons.push(lev);
}
let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20);
let mut body = Vec::new();
write!(&mut body, "[")?;
let mut first = true;
while let Some(document) = stream.next() {
let title_key = format!("{}-title", document.document_id);
let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
let description_key = format!("{}-description", document.document_id);
let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap();
let description = unsafe { from_utf8_unchecked(&description) };
let image_key = format!("{}-image", document.document_id);
let image = database.as_ref().get(image_key.as_bytes()).unwrap().unwrap();
let image = unsafe { from_utf8_unchecked(&image) };
let document = Document {
id: document.document_id,
title: title,
description: description,
image: image,
};
if !first { write!(&mut body, ",")? }
serde_json::to_writer(&mut body, &document)?;
first = false;
}
write!(&mut body, "]")?;
Ok(String::from_utf8(body)?)
}
fn main() {
let opt = Opt::from_args();
let name = opt.meta_name;
let map_file = format!("{}.map", name);
let idx_file = format!("{}.idx", name);
let sst_file = format!("{}.sst", name);
let rocksdb = "rocksdb/storage";
let meta = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
let meta = Arc::new(meta);
let db = DB::open_default(rocksdb).unwrap();
db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
drop(db);
let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
let db = Arc::new(db);
let common_words = common_words(opt.stop_words).expect("reading stop words");
let routes = warp::path("search")
.and(warp::query())
.map(move |query: SearchQuery| {
let body = search(meta.clone(), db.clone(), &common_words, &query.q).unwrap();
body
})
.with(warp::reply::with::header("Content-Type", "application/json"))
.with(warp::reply::with::header("Access-Control-Allow-Origin", "*"));
warp::serve(routes).run(opt.listen_addr);
}

3
raptor-indexer-csv/.gitignore vendored
View File

@ -1,3 +0,0 @@
/target
**/*.rs.bk

17
raptor-indexer-csv/Cargo.toml
View File

@ -1,17 +0,0 @@
[package]
edition = "2018"
name = "raptor-indexer-csv"
version = "0.1.0"
authors = ["Kerollmops <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "../raptor" }
serde = "1.0"
structopt = "0.2"
serde_derive = "1.0"
csv = "1.0"
unidecode = "0.3"
moby-name-gen = "0.1"
[dependencies.rocksdb]
git = "https://github.com/pingcap/rust-rocksdb.git"

143
raptor-indexer-csv/src/main.rs
View File

@ -1,143 +0,0 @@
// TODO make the raptor binary expose multiple subcommand
// make only one binary
#[macro_use] extern crate serde_derive;
use std::path::{Path, PathBuf};
use std::collections::{HashSet, BTreeMap};
use std::io::{self, BufReader, BufRead};
use std::fs::File;
use csv::ReaderBuilder;
use structopt::StructOpt;
use raptor::{MetadataBuilder, DocIndex, Tokenizer};
use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
use unidecode::unidecode;
#[derive(Debug, StructOpt)]
#[structopt(name = "raptor-indexer-csv", about = "A Raptor binary to index csv stored products.")]
struct Opt {
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
stop_words: PathBuf,
/// The csv file to index.
#[structopt(parse(from_os_str))]
products: PathBuf,
}
#[derive(Debug, Deserialize)]
struct Product {
#[serde(rename = "_unit_id")]
id: u64,
#[serde(rename = "product_title")]
title: String,
#[serde(rename = "product_image")]
image: String,
#[serde(rename = "product_description")]
description: String,
}
type CommonWords = HashSet<String>;
fn common_words<P>(path: P) -> io::Result<CommonWords>
where P: AsRef<Path>,
{
let file = File::open(path)?;
let file = BufReader::new(file);
let mut set = HashSet::new();
for line in file.lines().filter_map(|l| l.ok()) {
for word in line.split_whitespace() {
set.insert(word.to_owned());
}
}
Ok(set)
}
fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
where A: io::Write,
B: io::Write,
I: IntoIterator<Item=(usize, &'a str)>,
{
for (index, word) in words {
let doc_index = DocIndex {
document: doc_index,
attribute: attr,
attribute_index: index as u32,
};
// insert the exact representation
let word_lower = word.to_lowercase();
// and the unidecoded lowercased version
let word_unidecoded = unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
builder.insert(word_unidecoded, doc_index);
}
builder.insert(word_lower, doc_index);
}
}
fn main() {
let opt = Opt::from_args();
let common_words = common_words(opt.stop_words).expect("reading stop words");
// TODO add a subcommand to pack these files in a tar.xxx archive
let random_name = moby_name_gen::random_name();
let map_file = format!("{}.map", random_name);
let idx_file = format!("{}.idx", random_name);
let sst_file = format!("{}.sst", random_name);
let env_options = EnvOptions::new();
let cf_options = ColumnFamilyOptions::new();
let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
sst_file_writer.open(&sst_file).expect("open the sst file");
let map = File::create(&map_file).unwrap();
let indexes = File::create(&idx_file).unwrap();
let mut builder = MetadataBuilder::new(map, indexes);
let mut fields = BTreeMap::new();
let mut rdr = ReaderBuilder::new().from_path(opt.products).expect("reading product file");
let mut errors = 0;
for result in rdr.deserialize() {
let product: Product = match result {
Ok(product) => product,
Err(e) => { eprintln!("{:?}", e); errors += 1; continue },
};
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !common_words.contains(w));
insert_document_words(&mut builder, product.id, 0, title);
let description = Tokenizer::new(&product.description);
let description = description.iter().filter(|&(_, w)| !common_words.contains(w));
insert_document_words(&mut builder, product.id, 1, description);
// TODO simplify this by using functions and
// use the MetadataBuilder internal BTreeMap ?
let key = format!("{}-title", product.id);
let value = product.title;
fields.insert(key, value);
let key = format!("{}-description", product.id);
let value = product.description;
fields.insert(key, value);
let key = format!("{}-image", product.id);
let value = product.image;
fields.insert(key, value);
}
for (key, value) in fields {
sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
}
let _sst_file_info = sst_file_writer.finish().unwrap();
builder.finish().unwrap();
println!("Found {} errorneous lines", errors);
println!("Succesfully created {:?} dump.", random_name);
}

3
raptor-indexer/.gitignore vendored
View File

@ -1,3 +0,0 @@
/target
**/*.rs.bk

16
raptor-indexer/Cargo.toml
View File

@ -1,16 +0,0 @@
[package]
edition = "2018"
name = "raptor-indexer"
version = "0.1.0"
authors = ["Kerollmops <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "../raptor" }
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
unidecode = "0.3"
moby-name-gen = "0.1"
[dependencies.rocksdb]
git = "https://github.com/pingcap/rust-rocksdb.git"

123
raptor-indexer/src/main.rs
View File

@ -1,123 +0,0 @@
// TODO make the raptor binary expose multiple subcommand
// make only one binary
#[macro_use] extern crate serde_derive;
use std::path::Path;
use std::collections::{HashSet, BTreeMap};
use std::io::{self, BufReader, BufRead};
use std::fs::File;
use std::env;
use raptor::{MetadataBuilder, DocIndex, Tokenizer};
use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
use serde_json::from_str;
use unidecode::unidecode;
#[derive(Debug, Deserialize)]
struct Product {
title: String,
group_id: u64,
ft: String,
}
type CommonWords = HashSet<String>;
fn common_words<P>(path: P) -> io::Result<CommonWords>
where P: AsRef<Path>,
{
let file = File::open(path)?;
let file = BufReader::new(file);
let mut set = HashSet::new();
for line in file.lines().filter_map(|l| l.ok()) {
for word in line.split_whitespace() {
set.insert(word.to_owned());
}
}
Ok(set)
}
fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
where A: io::Write,
B: io::Write,
I: IntoIterator<Item=(usize, &'a str)>,
{
for (index, word) in words {
let doc_index = DocIndex {
document: doc_index,
attribute: attr,
attribute_index: index as u32,
};
// insert the exact representation
let word_lower = word.to_lowercase();
// and the unidecoded lowercased version
let word_unidecoded = unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
builder.insert(word_unidecoded, doc_index);
}
builder.insert(word_lower, doc_index);
}
}
fn main() {
let data_path = env::args().nth(1).expect("Missing data json lines file (e.g. products.json_lines)");
let data = File::open(data_path).unwrap();
let data = BufReader::new(data);
let common_path = "fr.stopwords.txt";
let common_words = common_words(common_path).unwrap_or_else(|e| {
println!("{:?}: {:?}", common_path, e);
HashSet::new()
});
// TODO add a subcommand to pack these files in a tar.xxx archive
let random_name = moby_name_gen::random_name();
let map_file = format!("{}.map", random_name);
let idx_file = format!("{}.idx", random_name);
let sst_file = format!("{}.sst", random_name);
let env_options = EnvOptions::new();
let cf_options = ColumnFamilyOptions::new();
let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
sst_file_writer.open(&sst_file).expect("open the sst file");
let map = File::create(&map_file).unwrap();
let indexes = File::create(&idx_file).unwrap();
let mut builder = MetadataBuilder::new(map, indexes);
let mut fields = BTreeMap::new();
for line in data.lines() {
let line = line.unwrap();
let product: Product = from_str(&line).unwrap();
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !common_words.contains(w));
insert_document_words(&mut builder, product.group_id, 0, title);
let description = Tokenizer::new(&product.ft);
let description = description.iter().filter(|&(_, w)| !common_words.contains(w));
insert_document_words(&mut builder, product.group_id, 1, description);
// TODO simplify this by using functions and
// use the MetadataBuilder internal BTreeMap ?
let key = format!("{}-title", product.group_id);
let value = product.title;
fields.insert(key, value);
let key = format!("{}-description", product.group_id);
let value = product.ft;
fields.insert(key, value);
}
for (key, value) in fields {
sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
}
let _sst_file_info = sst_file_writer.finish().unwrap();
builder.finish().unwrap();
println!("Succesfully created {:?} dump.", random_name);
}

3
raptor-search/.gitignore vendored
View File

@ -1,3 +0,0 @@
/target
**/*.rs.bk

17
raptor-search/Cargo.toml
View File

@ -1,17 +0,0 @@
[package]
edition = "2018"
name = "raptor-search"
version = "0.1.0"
authors = ["Kerollmops <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "../raptor" }
structopt = "0.2"
elapsed = "0.1"
[dependencies.fst]
git = "https://github.com/Kerollmops/fst.git"
branch = "automaton-for-deref"
[dependencies.rocksdb]
git = "https://github.com/pingcap/rust-rocksdb.git"

29
raptor-search/benches/main.rs
View File

@ -1,29 +0,0 @@
#![feature(test)]
extern crate test;
extern crate fst;
extern crate raptor;
use fst::Streamer;
use raptor::{load_map, RankedStream, LevBuilder};
#[bench]
fn chauve_souris(b: &mut test::Bencher) {
let lev_builder = LevBuilder::new();
let map = load_map("map.fst", "values.vecs").unwrap();
let query = "chauve souris";
b.iter(|| {
let mut automatons = Vec::new();
for query in query.split_whitespace() {
let lev = lev_builder.build_automaton(query);
automatons.push(lev);
}
let mut stream = RankedStream::new(&map, &map.values(), automatons, 20);
while let Some(document_id) = stream.next() {
test::black_box(document_id);
}
})
}

97
raptor-search/src/main.rs
View File

@ -1,97 +0,0 @@
use std::fs::File;
use std::path::{Path, PathBuf};
use std::collections::HashSet;
use std::str::from_utf8_unchecked;
use std::io::{self, BufReader, BufRead, Write};
use structopt::StructOpt;
use elapsed::measure_time;
use fst::Streamer;
use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
use raptor::{automaton, Metadata, RankedStream};
#[derive(Debug, StructOpt)]
#[structopt(name = "raptor-search", about = "A Raptor binary to search in a dump.")]
struct Opt {
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
stop_words: PathBuf,
/// Meta file name (e.g. relaxed-colden).
meta_name: String,
}
type CommonWords = HashSet<String>;
fn common_words<P>(path: P) -> io::Result<CommonWords>
where P: AsRef<Path>,
{
let file = File::open(path)?;
let file = BufReader::new(file);
let mut set = HashSet::new();
for line in file.lines().filter_map(|l| l.ok()) {
for word in line.split_whitespace() {
set.insert(word.to_owned());
}
}
Ok(set)
}
fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) {
let mut automatons = Vec::new();
for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) {
let lev = automaton::build(query);
automatons.push(lev);
}
let mut stream = RankedStream::new(&metadata, automatons, 20);
while let Some(document) = stream.next() {
print!("{:?}", document.document_id);
let title_key = format!("{}-title", document.document_id);
let title = database.get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
print!(" {:?}", title);
println!();
}
}
fn main() {
let opt = Opt::from_args();
let name = opt.meta_name;
let map_file = format!("{}.map", name);
let idx_file = format!("{}.idx", name);
let sst_file = format!("{}.sst", name);
let rocksdb = "rocksdb/storage";
let (elapsed, meta) = measure_time(|| unsafe {
Metadata::from_paths(map_file, idx_file).unwrap()
});
println!("{} to load metadata", elapsed);
let (elapsed, db) = measure_time(|| {
let db = DB::open_default(rocksdb).unwrap();
db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
drop(db);
DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap()
});
println!("{} to load the SST file in RocksDB and reopen it for read-only", elapsed);
let common_words = common_words(opt.stop_words).expect("reading stop words");
loop {
print!("Searching for: ");
io::stdout().flush().unwrap();
let mut query = String::new();
io::stdin().read_line(&mut query).unwrap();
let query = query.trim().to_lowercase();
if query.is_empty() { break }
let (elapsed, _) = measure_time(|| search(&meta, &db, &common_words, &query));
println!("Finished in {}", elapsed);
}
}

20
src/common_words.rs Normal file
View File

@ -0,0 +1,20 @@
use std::io::{self, BufReader, BufRead};
use std::collections::HashSet;
use std::path::Path;
use std::fs::File;
pub type CommonWords = HashSet<String>;
pub fn from_file<P>(path: P) -> io::Result<CommonWords>
where P: AsRef<Path>,
{
let file = File::open(path)?;
let file = BufReader::new(file);
let mut set = HashSet::new();
for line in file.lines().filter_map(|l| l.ok()) {
for word in line.split_whitespace() {
set.insert(word.to_owned());
}
}
Ok(set)
}

122
src/index/csv.rs Normal file
View File

@ -0,0 +1,122 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::fs::File;
use std::io;
use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
use raptor::{MetadataBuilder, DocIndex, Tokenizer};
use unidecode::unidecode;
use csv::ReaderBuilder;
use crate::common_words::{self, CommonWords};
use crate::index::csv_feature::CommandCsv;
#[derive(Debug, Deserialize)]
struct Product {
#[serde(rename = "_unit_id")]
id: u64,
#[serde(rename = "product_title")]
title: String,
#[serde(rename = "product_image")]
image: String,
#[serde(rename = "product_description")]
description: String,
}
#[derive(Debug)]
pub struct CsvIndexer {
common_words: CommonWords,
products: PathBuf,
}
impl CsvIndexer {
pub fn from_command(command: CommandCsv) -> io::Result<CsvIndexer> {
let common_words = common_words::from_file(command.stop_words)?;
let products = command.products;
Ok(CsvIndexer { common_words, products })
}
pub fn index(self) {
let random_name = moby_name_gen::random_name();
let map_file = format!("{}.map", random_name);
let idx_file = format!("{}.idx", random_name);
let sst_file = format!("{}.sst", random_name);
let env_options = EnvOptions::new();
let cf_options = ColumnFamilyOptions::new();
let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
sst_file_writer.open(&sst_file).expect("open the sst file");
let map = File::create(&map_file).unwrap();
let indexes = File::create(&idx_file).unwrap();
let mut builder = MetadataBuilder::new(map, indexes);
let mut fields = BTreeMap::new();
let mut rdr = ReaderBuilder::new().from_path(&self.products).expect("reading product file");
let mut errors = 0;
for result in rdr.deserialize() {
let product: Product = match result {
Ok(product) => product,
Err(e) => { eprintln!("{:?}", e); errors += 1; continue },
};
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 0, title);
let description = Tokenizer::new(&product.description);
let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 1, description);
// TODO simplify this by using functions and
// use the MetadataBuilder internal BTreeMap ?
let key = format!("{}-title", product.id);
let value = product.title;
fields.insert(key, value);
let key = format!("{}-description", product.id);
let value = product.description;
fields.insert(key, value);
let key = format!("{}-image", product.id);
let value = product.image;
fields.insert(key, value);
}
for (key, value) in fields {
sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
}
let _sst_file_info = sst_file_writer.finish().unwrap();
builder.finish().unwrap();
println!("Found {} errorneous lines", errors);
println!("Succesfully created {:?} dump.", random_name);
}
}
fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
where A: io::Write,
B: io::Write,
I: IntoIterator<Item=(usize, &'a str)>,
{
for (index, word) in words {
let doc_index = DocIndex {
document: doc_index,
attribute: attr,
attribute_index: index as u32,
};
// insert the exact representation
let word_lower = word.to_lowercase();
// and the unidecoded lowercased version
let word_unidecoded = unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
builder.insert(word_unidecoded, doc_index);
}
builder.insert(word_lower, doc_index);
}
}

112
src/index/json_lines.rs Normal file
View File

@ -0,0 +1,112 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::fs::File;
use std::io::{self, BufReader, BufRead};
use serde_json::from_str;
use rocksdb::{SstFileWriter, EnvOptions, ColumnFamilyOptions};
use raptor::{MetadataBuilder, DocIndex, Tokenizer};
use unidecode::unidecode;
use crate::common_words::{self, CommonWords};
use crate::index::jsonlines_feature::CommandJsonLines;
#[derive(Debug, Deserialize)]
struct Product {
title: String,
group_id: u64,
ft: String,
}
#[derive(Debug)]
pub struct JsonLinesIndexer {
common_words: CommonWords,
products: PathBuf,
}
impl JsonLinesIndexer {
pub fn from_command(command: CommandJsonLines) -> io::Result<JsonLinesIndexer> {
let common_words = common_words::from_file(command.stop_words)?;
let products = command.products;
Ok(JsonLinesIndexer { common_words, products })
}
pub fn index(self) {
let data = File::open(&self.products).unwrap();
let data = BufReader::new(data);
// TODO add a subcommand to pack these files in a tar.xxx archive
let random_name = moby_name_gen::random_name();
let map_file = format!("{}.map", random_name);
let idx_file = format!("{}.idx", random_name);
let sst_file = format!("{}.sst", random_name);
let env_options = EnvOptions::new();
let cf_options = ColumnFamilyOptions::new();
let mut sst_file_writer = SstFileWriter::new(env_options, cf_options);
sst_file_writer.open(&sst_file).expect("open the sst file");
let map = File::create(&map_file).unwrap();
let indexes = File::create(&idx_file).unwrap();
let mut builder = MetadataBuilder::new(map, indexes);
let mut fields = BTreeMap::new();
for line in data.lines() {
let line = line.unwrap();
let product: Product = from_str(&line).unwrap();
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.group_id, 0, title);
let description = Tokenizer::new(&product.ft);
let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.group_id, 1, description);
// TODO simplify this by using functions and
// use the MetadataBuilder internal BTreeMap ?
let key = format!("{}-title", product.group_id);
let value = product.title;
fields.insert(key, value);
let key = format!("{}-description", product.group_id);
let value = product.ft;
fields.insert(key, value);
}
for (key, value) in fields {
sst_file_writer.put(key.as_bytes(), value.as_bytes()).unwrap();
}
let _sst_file_info = sst_file_writer.finish().unwrap();
builder.finish().unwrap();
println!("Succesfully created {:?} dump.", random_name);
}
}
fn insert_document_words<'a, I, A, B>(builder: &mut MetadataBuilder<A, B>, doc_index: u64, attr: u8, words: I)
where A: io::Write,
B: io::Write,
I: IntoIterator<Item=(usize, &'a str)>,
{
for (index, word) in words {
let doc_index = DocIndex {
document: doc_index,
attribute: attr,
attribute_index: index as u32,
};
// insert the exact representation
let word_lower = word.to_lowercase();
// and the unidecoded lowercased version
let word_unidecoded = unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
builder.insert(word_unidecoded, doc_index);
}
builder.insert(word_lower, doc_index);
}
}

71
src/index/mod.rs Normal file
View File

@ -0,0 +1,71 @@
#[cfg(feature = "index-csv")]
mod csv;
#[cfg(feature = "index-jsonlines")]
mod json_lines;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub enum CommandIndex {
#[cfg(feature = "index-jsonlines")]
/// Index files encoded as json lines.
#[structopt(name = "json-lines")]
JsonLines(self::jsonlines_feature::CommandJsonLines),
#[cfg(feature = "index-csv")]
/// Index files encoded as csv.
#[structopt(name = "csv")]
Csv(self::csv_feature::CommandCsv),
}
#[cfg(feature = "index-jsonlines")]
pub mod jsonlines_feature {
use std::error;
use std::path::PathBuf;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub struct CommandJsonLines {
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
pub stop_words: PathBuf,
/// The csv file to index.
#[structopt(parse(from_os_str))]
pub products: PathBuf,
}
pub fn json_lines(command: CommandJsonLines) -> Result<(), Box<error::Error>> {
use super::json_lines::JsonLinesIndexer;
let indexer = JsonLinesIndexer::from_command(command)?;
Ok(indexer.index())
}
}
#[cfg(feature = "index-csv")]
pub mod csv_feature {
use std::error;
use std::path::PathBuf;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub struct CommandCsv {
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
pub stop_words: PathBuf,
/// The csv file to index.
#[structopt(parse(from_os_str))]
pub products: PathBuf,
}
pub fn csv(command: CommandCsv) -> Result<(), Box<error::Error>> {
use super::csv::CsvIndexer;
let indexer = CsvIndexer::from_command(command)?;
Ok(indexer.index())
}
}

50
src/main.rs Normal file
View File

@ -0,0 +1,50 @@
#[macro_use] extern crate serde_derive;
#[cfg(feature = "index")]
mod index;
#[cfg(feature = "serve")]
mod serve;
mod common_words;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
#[structopt(name = "raptor-cli", about = "A command line to do raptor operations.")]
enum Commands {
#[cfg(feature = "index")]
/// Index files of different format.
#[structopt(name = "index")]
Index(index::CommandIndex),
#[cfg(feature = "serve")]
/// Serve indexes.
#[structopt(name = "serve")]
Serve(serve::CommandServe),
}
fn main() {
let ret = match Commands::from_args() {
#[cfg(feature = "index")]
Commands::Index(i) => match i {
#[cfg(feature = "index-jsonlines")]
index::CommandIndex::JsonLines(command) => index::jsonlines_feature::json_lines(command),
#[cfg(feature = "index-csv")]
index::CommandIndex::Csv(command) => index::csv_feature::csv(command),
},
#[cfg(feature = "serve")]
Commands::Serve(s) => match s {
#[cfg(feature = "serve-http")]
serve::CommandServe::Http(command) => serve::http_feature::http(command),
#[cfg(feature = "serve-console")]
serve::CommandServe::Console(command) => serve::console_feature::console(command),
},
};
if let Err(e) = ret { eprintln!("{}", e) }
}

72
src/serve/console.rs Normal file
View File

@ -0,0 +1,72 @@
use std::str::from_utf8_unchecked;
use std::io::{self, Write};
use fst::Streamer;
use elapsed::measure_time;
use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
use raptor::{automaton, Metadata, RankedStream};
use crate::serve::console_feature::CommandConsole;
use crate::common_words::{self, CommonWords};
pub struct ConsoleSearch {
common_words: CommonWords,
metadata: Metadata,
db: DB,
}
impl ConsoleSearch {
pub fn from_command(command: CommandConsole) -> io::Result<ConsoleSearch> {
let common_words = common_words::from_file(command.stop_words)?;
let meta_name = command.meta_name.display();
let map_file = format!("{}.map", meta_name);
let idx_file = format!("{}.idx", meta_name);
let sst_file = format!("{}.sst", meta_name);
let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
let rocksdb = "rocksdb/storage";
let db = DB::open_default(rocksdb).unwrap();
db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
drop(db);
let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
Ok(ConsoleSearch { common_words, metadata, db })
}
pub fn serve(self) {
loop {
print!("Searching for: ");
io::stdout().flush().unwrap();
let mut query = String::new();
io::stdin().read_line(&mut query).unwrap();
let query = query.trim().to_lowercase();
if query.is_empty() { break }
let (elapsed, _) = measure_time(|| search(&self.metadata, &self.db, &self.common_words, &query));
println!("Finished in {}", elapsed);
}
}
}
fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) {
let mut automatons = Vec::new();
for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) {
let lev = automaton::build(query);
automatons.push(lev);
}
let mut stream = RankedStream::new(&metadata, automatons, 20);
while let Some(document) = stream.next() {
print!("{:?}", document.document_id);
let title_key = format!("{}-title", document.document_id);
let title = database.get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
print!(" {:?}", title);
println!();
}
}

120
src/serve/http.rs Normal file
View File

@ -0,0 +1,120 @@
use std::str::from_utf8_unchecked;
use std::io::{self, Write};
use std::net::SocketAddr;
use std::error::Error;
use std::sync::Arc;
use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
use raptor::{automaton, Metadata};
use raptor::rank::RankedStream;
use fst::Streamer;
use warp::Filter;
use crate::serve::http_feature::CommandHttp;
use crate::common_words::{self, CommonWords};
#[derive(Debug, Serialize)]
struct Document<'a> {
id: u64,
title: &'a str,
description: &'a str,
image: &'a str,
}
pub struct HttpServer {
listen_addr: SocketAddr,
common_words: Arc<CommonWords>,
metadata: Arc<Metadata>,
db: Arc<DB>,
}
#[derive(Debug, Deserialize)]
struct SearchQuery { q: String }
impl HttpServer {
pub fn from_command(command: CommandHttp) -> io::Result<HttpServer> {
let common_words = common_words::from_file(command.stop_words)?;
let meta_name = command.meta_name.display();
let map_file = format!("{}.map", meta_name);
let idx_file = format!("{}.idx", meta_name);
let sst_file = format!("{}.sst", meta_name);
let metadata = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
let rocksdb = "rocksdb/storage";
let db = DB::open_default(rocksdb).unwrap();
db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
drop(db);
let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
Ok(HttpServer {
listen_addr: command.listen_addr,
common_words: Arc::new(common_words),
metadata: Arc::new(metadata),
db: Arc::new(db),
})
}
pub fn serve(self) {
let HttpServer { listen_addr, common_words, metadata, db } = self;
let routes = warp::path("search")
.and(warp::query())
.map(move |query: SearchQuery| {
let body = search(metadata.clone(), db.clone(), common_words.clone(), &query.q).unwrap();
body
})
.with(warp::reply::with::header("Content-Type", "application/json"))
.with(warp::reply::with::header("Access-Control-Allow-Origin", "*"));
warp::serve(routes).run(listen_addr)
}
}
fn search<M, D, C>(metadata: M, database: D, common_words: C, query: &str) -> Result<String, Box<Error>>
where M: AsRef<Metadata>,
D: AsRef<DB>,
C: AsRef<CommonWords>,
{
let mut automatons = Vec::new();
for query in query.split_whitespace().map(str::to_lowercase) {
if common_words.as_ref().contains(&query) { continue }
let lev = automaton::build(&query);
automatons.push(lev);
}
let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20);
let mut body = Vec::new();
write!(&mut body, "[")?;
let mut first = true;
while let Some(document) = stream.next() {
let title_key = format!("{}-title", document.document_id);
let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
let description_key = format!("{}-description", document.document_id);
let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap();
let description = unsafe { from_utf8_unchecked(&description) };
let image_key = format!("{}-image", document.document_id);
let image = database.as_ref().get(image_key.as_bytes()).unwrap().unwrap();
let image = unsafe { from_utf8_unchecked(&image) };
let document = Document {
id: document.document_id,
title: title,
description: description,
image: image,
};
if !first { write!(&mut body, ",")? }
serde_json::to_writer(&mut body, &document)?;
first = false;
}
write!(&mut body, "]")?;
Ok(String::from_utf8(body)?)
}

76
src/serve/mod.rs Normal file
View File

@ -0,0 +1,76 @@
#[cfg(feature = "serve-http")]
mod http;
#[cfg(feature = "serve-console")]
mod console;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub enum CommandServe {
#[cfg(feature = "serve-http")]
/// Serve an index under an http protocol.
#[structopt(name = "http")]
Http(self::http_feature::CommandHttp),
#[cfg(feature = "serve-console")]
/// Serve an index under a simple console.
#[structopt(name = "console")]
Console(self::console_feature::CommandConsole),
}
#[cfg(feature = "serve-http")]
pub mod http_feature {
use std::error;
use std::path::PathBuf;
use std::net::SocketAddr;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub struct CommandHttp {
/// The address and port to bind the server to.
#[structopt(short = "l", default_value = "127.0.0.1:3030")]
pub listen_addr: SocketAddr,
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
pub stop_words: PathBuf,
/// Meta file name (e.g. relaxed-colden).
#[structopt(parse(from_os_str))]
pub meta_name: PathBuf,
}
pub fn http(command: CommandHttp) -> Result<(), Box<error::Error>> {
use super::http::HttpServer;
let server = HttpServer::from_command(command)?;
Ok(server.serve())
}
}
#[cfg(feature = "serve-console")]
pub mod console_feature {
use std::error;
use std::path::PathBuf;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
pub struct CommandConsole {
/// The stop word file, each word must be separated by a newline.
#[structopt(long = "stop-words", parse(from_os_str))]
pub stop_words: PathBuf,
/// Meta file name (e.g. relaxed-colden).
#[structopt(parse(from_os_str))]
pub meta_name: PathBuf,
}
pub fn console(command: CommandConsole) -> Result<(), Box<error::Error>> {
use super::console::ConsoleSearch;
let search = ConsoleSearch::from_command(command)?;
Ok(search.serve())
}
}