mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-29 16:24:26 +01:00
clean: Remove statics and use Arc
s
This commit is contained in:
parent
a18afdc0f1
commit
1e14df00d1
10
Cargo.lock
generated
10
Cargo.lock
generated
@ -21,6 +21,14 @@ dependencies = [
|
|||||||
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "levenshtein_automata"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst#5e8183a7634c4a0182ea7bb398140b2fe9854f77"
|
||||||
|
dependencies = [
|
||||||
|
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.40"
|
version = "0.2.40"
|
||||||
@ -57,6 +65,7 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||||
|
"levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)",
|
||||||
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
]
|
]
|
||||||
@ -114,6 +123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
"checksum bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bda13183df33055cbb84b847becce220d392df502ebe7a4a78d7021771ed94d0"
|
"checksum bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bda13183df33055cbb84b847becce220d392df502ebe7a4a78d7021771ed94d0"
|
||||||
"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87"
|
"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87"
|
||||||
"checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "<none>"
|
"checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "<none>"
|
||||||
|
"checksum levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)" = "<none>"
|
||||||
"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b"
|
"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b"
|
||||||
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
|
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
|
||||||
"checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7"
|
"checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7"
|
||||||
|
@ -11,3 +11,8 @@ serde_derive = "1.0"
|
|||||||
[dependencies.fst]
|
[dependencies.fst]
|
||||||
git = "https://github.com/Kerollmops/fst.git"
|
git = "https://github.com/Kerollmops/fst.git"
|
||||||
branch = "op-builder-with-state"
|
branch = "op-builder-with-state"
|
||||||
|
|
||||||
|
[dependencies.levenshtein_automata]
|
||||||
|
git = "https://github.com/Kerollmops/levenshtein-automata.git"
|
||||||
|
branch = "custom-fst"
|
||||||
|
features = ["fst_automaton"]
|
||||||
|
12
raptor-http/Cargo.lock
generated
12
raptor-http/Cargo.lock
generated
@ -80,6 +80,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "fst"
|
name = "fst"
|
||||||
version = "0.3.0"
|
version = "0.3.0"
|
||||||
|
source = "git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state#6e0ab4e4ee5443cc55079996bf9f703086322c33"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
"byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
@ -150,8 +151,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "levenshtein_automata"
|
name = "levenshtein_automata"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
source = "git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst#5e8183a7634c4a0182ea7bb398140b2fe9854f77"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fst 0.3.0",
|
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -292,7 +294,8 @@ name = "raptor"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"fst 0.3.0",
|
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||||
|
"levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)",
|
||||||
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
]
|
]
|
||||||
@ -302,9 +305,8 @@ name = "raptor-http"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"fst 0.3.0",
|
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||||
"futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
|
"futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"levenshtein_automata 0.1.0",
|
|
||||||
"raptor 0.1.0",
|
"raptor 0.1.0",
|
||||||
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
"serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
@ -627,6 +629,7 @@ dependencies = [
|
|||||||
"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81"
|
"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81"
|
||||||
"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b"
|
"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b"
|
||||||
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
|
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
|
||||||
|
"checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "<none>"
|
||||||
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
||||||
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
||||||
"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c"
|
"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c"
|
||||||
@ -636,6 +639,7 @@ dependencies = [
|
|||||||
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
||||||
"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
|
"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
|
||||||
"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef"
|
"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef"
|
||||||
|
"checksum levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)" = "<none>"
|
||||||
"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b"
|
"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b"
|
||||||
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
|
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
|
||||||
"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2"
|
"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2"
|
||||||
|
@ -14,11 +14,6 @@ tokio-proto = "0.1"
|
|||||||
tokio-service = "0.1"
|
tokio-service = "0.1"
|
||||||
url = "1.7"
|
url = "1.7"
|
||||||
|
|
||||||
[dependencies.levenshtein_automata]
|
|
||||||
git = "https://github.com/Kerollmops/levenshtein-automata.git"
|
|
||||||
branch = "custom-fst"
|
|
||||||
features = ["fst"]
|
|
||||||
|
|
||||||
[dependencies.fst]
|
[dependencies.fst]
|
||||||
git = "https://github.com/Kerollmops/fst.git"
|
git = "https://github.com/Kerollmops/fst.git"
|
||||||
branch = "op-builder-with-state"
|
branch = "op-builder-with-state"
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
extern crate env_logger;
|
extern crate env_logger;
|
||||||
extern crate fst;
|
extern crate fst;
|
||||||
extern crate futures;
|
extern crate futures;
|
||||||
extern crate levenshtein_automata;
|
|
||||||
extern crate raptor;
|
extern crate raptor;
|
||||||
extern crate tokio_minihttp;
|
extern crate tokio_minihttp;
|
||||||
extern crate tokio_proto;
|
extern crate tokio_proto;
|
||||||
@ -9,29 +8,22 @@ extern crate tokio_service;
|
|||||||
extern crate url;
|
extern crate url;
|
||||||
|
|
||||||
use std::{io, fs};
|
use std::{io, fs};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use fst::Streamer;
|
use fst::Streamer;
|
||||||
use futures::future;
|
use futures::future;
|
||||||
use levenshtein_automata::LevenshteinAutomatonBuilder as LevBuilder;
|
|
||||||
use tokio_minihttp::{Request, Response, Http};
|
use tokio_minihttp::{Request, Response, Http};
|
||||||
use tokio_proto::TcpServer;
|
use tokio_proto::TcpServer;
|
||||||
use tokio_service::Service;
|
use tokio_service::Service;
|
||||||
|
|
||||||
use raptor::{Map, OpWithStateBuilder};
|
use raptor::{Map, OpWithStateBuilder, LevBuilder};
|
||||||
|
|
||||||
static mut MAP: Option<Map<u64>> = None;
|
struct MainService {
|
||||||
static mut LEV_BUILDER_0: Option<LevBuilder> = None;
|
map: Arc<Map<u64>>,
|
||||||
static mut LEV_BUILDER_1: Option<LevBuilder> = None;
|
lev_builder: Arc<LevBuilder>,
|
||||||
static mut LEV_BUILDER_2: Option<LevBuilder> = None;
|
|
||||||
|
|
||||||
struct MainService<'a> {
|
|
||||||
map: &'a Map<u64>,
|
|
||||||
lev_builder_0: &'a LevBuilder,
|
|
||||||
lev_builder_1: &'a LevBuilder,
|
|
||||||
lev_builder_2: &'a LevBuilder,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> Service for MainService<'a> {
|
impl Service for MainService {
|
||||||
type Request = Request;
|
type Request = Request;
|
||||||
type Response = Response;
|
type Response = Response;
|
||||||
type Error = io::Error;
|
type Error = io::Error;
|
||||||
@ -52,13 +44,7 @@ impl<'a> Service for MainService<'a> {
|
|||||||
let mut automatons = Vec::new();
|
let mut automatons = Vec::new();
|
||||||
|
|
||||||
for query in query.split_whitespace() {
|
for query in query.split_whitespace() {
|
||||||
let lev = if query.len() <= 4 {
|
let lev = self.lev_builder.build_automaton(query);
|
||||||
self.lev_builder_0.build_dfa(&query)
|
|
||||||
} else if query.len() <= 8 {
|
|
||||||
self.lev_builder_1.build_dfa(&query)
|
|
||||||
} else {
|
|
||||||
self.lev_builder_2.build_dfa(&query)
|
|
||||||
};
|
|
||||||
automatons.push(lev);
|
automatons.push(lev);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -100,27 +86,18 @@ impl<'a> Service for MainService<'a> {
|
|||||||
fn main() {
|
fn main() {
|
||||||
drop(env_logger::init());
|
drop(env_logger::init());
|
||||||
|
|
||||||
// initialize all static variables
|
|
||||||
unsafe {
|
|
||||||
MAP = {
|
|
||||||
let map = fs::read("map.fst").unwrap();
|
|
||||||
let values = fs::read("values.vecs").unwrap();
|
|
||||||
|
|
||||||
Some(Map::from_bytes(map, &values).unwrap())
|
|
||||||
};
|
|
||||||
LEV_BUILDER_0 = Some(LevBuilder::new(0, false));
|
|
||||||
LEV_BUILDER_1 = Some(LevBuilder::new(1, false));
|
|
||||||
LEV_BUILDER_2 = Some(LevBuilder::new(2, false));
|
|
||||||
}
|
|
||||||
|
|
||||||
let addr = "0.0.0.0:8080".parse().unwrap();
|
let addr = "0.0.0.0:8080".parse().unwrap();
|
||||||
|
|
||||||
unsafe {
|
let lev_builder = Arc::new(LevBuilder::new());
|
||||||
TcpServer::new(Http, addr).serve(|| Ok(MainService {
|
let map = {
|
||||||
map: MAP.as_ref().unwrap(),
|
let fst = fs::read("map.fst").unwrap();
|
||||||
lev_builder_0: LEV_BUILDER_0.as_ref().unwrap(),
|
let values = fs::read("values.vecs").unwrap();
|
||||||
lev_builder_1: LEV_BUILDER_1.as_ref().unwrap(),
|
let map = Map::from_bytes(fst, &values).unwrap();
|
||||||
lev_builder_2: LEV_BUILDER_2.as_ref().unwrap(),
|
Arc::new(map)
|
||||||
}))
|
};
|
||||||
}
|
|
||||||
|
TcpServer::new(Http, addr).serve(move || Ok(MainService {
|
||||||
|
map: map.clone(),
|
||||||
|
lev_builder: lev_builder.clone(),
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
27
src/levenshtein.rs
Normal file
27
src/levenshtein.rs
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
|
||||||
|
|
||||||
|
pub struct LevBuilder {
|
||||||
|
automatons: [LevenshteinAutomatonBuilder; 3],
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LevBuilder {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
automatons: [
|
||||||
|
LevenshteinAutomatonBuilder::new(0, false),
|
||||||
|
LevenshteinAutomatonBuilder::new(1, false),
|
||||||
|
LevenshteinAutomatonBuilder::new(2, false),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_automaton(&self, query: &str) -> DFA {
|
||||||
|
if query.len() <= 4 {
|
||||||
|
self.automatons[0].build_dfa(query)
|
||||||
|
} else if query.len() <= 8 {
|
||||||
|
self.automatons[1].build_dfa(query)
|
||||||
|
} else {
|
||||||
|
self.automatons[2].build_dfa(query)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,12 +1,16 @@
|
|||||||
#[macro_use] extern crate serde_derive;
|
#[macro_use] extern crate serde_derive;
|
||||||
extern crate bincode;
|
extern crate bincode;
|
||||||
extern crate fst;
|
extern crate fst;
|
||||||
|
extern crate levenshtein_automata;
|
||||||
extern crate serde;
|
extern crate serde;
|
||||||
|
|
||||||
pub mod map;
|
pub mod map;
|
||||||
|
mod levenshtein;
|
||||||
|
|
||||||
pub use self::map::{Map, MapBuilder, Values};
|
pub use self::map::{Map, MapBuilder, Values};
|
||||||
pub use self::map::{
|
pub use self::map::{
|
||||||
OpBuilder, IndexedValues,
|
OpBuilder, IndexedValues,
|
||||||
OpWithStateBuilder, IndexedValuesWithState,
|
OpWithStateBuilder, IndexedValuesWithState,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub use self::levenshtein::LevBuilder;
|
||||||
|
Loading…
Reference in New Issue
Block a user