feat: Fix the ranking algorithm to sort only the needed documents
This commit is contained in:
Kerollmops
Clément Renault
parent
0190caef4d
commit
23134fee02
|
|
@ -4,7 +4,7 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -24,7 +24,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "group-by"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392"
|
||||
|
||||
[[package]]
|
||||
name = "levenshtein_automata"
|
||||
|
|
@ -72,18 +72,18 @@ dependencies = [
|
|||
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||
"group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)",
|
||||
"levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -135,8 +135,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
|
||||
"checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6"
|
||||
"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035"
|
||||
"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95"
|
||||
"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79"
|
||||
"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e"
|
||||
"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe"
|
||||
"checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d"
|
||||
"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -29,7 +29,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "group-by"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
|
|
@ -82,8 +82,8 @@ dependencies = [
|
|||
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||
"group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)",
|
||||
"levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -91,20 +91,20 @@ name = "raptor-indexer"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"raptor 0.1.0",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -119,7 +119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
dependencies = [
|
||||
"dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -173,8 +173,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
|
||||
"checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6"
|
||||
"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035"
|
||||
"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95"
|
||||
"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79"
|
||||
"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e"
|
||||
"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe"
|
||||
"checksum serde_json 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)" = "84b8035cabe9b35878adec8ac5fe03d5f6bc97ff6edd7ccb96b44c1276ba390e"
|
||||
"checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d"
|
||||
"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@ where P: AsRef<Path>
|
|||
fs::set_permissions(&path, perms)
|
||||
}
|
||||
|
||||
fn is_readonly<P>(path: P) -> io::Result<bool>
|
||||
where P: AsRef<Path>
|
||||
{
|
||||
fs::metadata(&path).map(|m| m.permissions().readonly())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data = File::open("products.json_lines").unwrap();
|
||||
let data = BufReader::new(data);
|
||||
|
|
@ -54,6 +60,18 @@ fn main() {
|
|||
}
|
||||
};
|
||||
|
||||
let map_file = "map.fst";
|
||||
let values_file = "values.vecs";
|
||||
|
||||
for file in &[map_file, values_file] {
|
||||
match is_readonly(file) {
|
||||
Ok(true) => panic!("the {:?} file is readonly, please make it writeable", file),
|
||||
Err(ref e) if e.kind() == io::ErrorKind::NotFound => (),
|
||||
Err(e) => panic!("{:?}", e),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
let mut builder = DocIndexMapBuilder::new();
|
||||
for line in data.lines() {
|
||||
let line = line.unwrap();
|
||||
|
|
@ -83,13 +101,13 @@ fn main() {
|
|||
}
|
||||
}
|
||||
|
||||
let map = File::create("map.fst").unwrap();
|
||||
let values = File::create("values.vecs").unwrap();
|
||||
let map = File::create(map_file).unwrap();
|
||||
let values = File::create(values_file).unwrap();
|
||||
|
||||
let (map, values) = builder.build(map, values).unwrap();
|
||||
|
||||
set_readonly("map.fst", true).unwrap();
|
||||
set_readonly("values.vecs", true).unwrap();
|
||||
set_readonly(map_file, true).unwrap();
|
||||
set_readonly(values_file, true).unwrap();
|
||||
|
||||
println!("Checking the dump consistency...");
|
||||
unsafe { DocIndexMap::from_paths("map.fst", "values.vecs").unwrap() };
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -14,7 +14,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.3"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
|
|
@ -42,7 +42,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "group-by"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec"
|
||||
source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392"
|
||||
|
||||
[[package]]
|
||||
name = "levenshtein_automata"
|
||||
|
|
@ -70,7 +70,7 @@ name = "log"
|
|||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -106,8 +106,8 @@ dependencies = [
|
|||
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||
"group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)",
|
||||
"levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -118,18 +118,18 @@ dependencies = [
|
|||
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)",
|
||||
"raptor 0.1.0",
|
||||
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -174,7 +174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
[metadata]
|
||||
"checksum bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9f2fb9e29e72fd6bc12071533d5dc7664cb01480c59406f656d7ac25c7bd8ff7"
|
||||
"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9"
|
||||
"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18"
|
||||
"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e"
|
||||
"checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29"
|
||||
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
|
||||
"checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "<none>"
|
||||
|
|
@ -186,8 +186,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
|
||||
"checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6"
|
||||
"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035"
|
||||
"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95"
|
||||
"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79"
|
||||
"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e"
|
||||
"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe"
|
||||
"checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d"
|
||||
"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@ extern crate fst;
|
|||
extern crate raptor;
|
||||
extern crate elapsed;
|
||||
|
||||
use std::{env, fs};
|
||||
use std::process::Command;
|
||||
use std::env;
|
||||
use std::io::{self, Write};
|
||||
use elapsed::measure_time;
|
||||
use fst::Streamer;
|
||||
|
|
@ -21,7 +20,8 @@ fn search(map: &DocIndexMap, lev_builder: &LevBuilder, query: &str) {
|
|||
while let Some(document_id) = stream.next() {
|
||||
print!("{:?}", document_id);
|
||||
|
||||
/* only here to debug !
|
||||
// /* only here to debug !
|
||||
use std::{fs, process::Command};
|
||||
if let Ok(_) = fs::File::open("products.json_lines") {
|
||||
let output = Command::new("rg")
|
||||
.arg(document_id.to_string())
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ impl<T> Values<T> {
|
|||
let cap = raw.iter().map(Vec::len).sum();
|
||||
let mut values = Vec::with_capacity(cap);
|
||||
|
||||
for v in &raw {
|
||||
for mut v in &raw {
|
||||
let len = v.len() as u64;
|
||||
let start = ranges.last().map(|&Range { end, .. }| end).unwrap_or(0);
|
||||
|
||||
|
|
|
|||
288
src/rank.rs
288
src/rank.rs
|
|
@ -1,6 +1,6 @@
|
|||
use std::cmp::{self, Ordering};
|
||||
use std::{mem, vec};
|
||||
use std::collections::{HashSet, HashMap};
|
||||
use std::collections::HashMap;
|
||||
use std::{mem, vec, iter};
|
||||
use DocIndexMap;
|
||||
use fst;
|
||||
use levenshtein_automata::DFA;
|
||||
|
|
@ -10,11 +10,16 @@ use map::{
|
|||
Values,
|
||||
};
|
||||
use {Match, DocIndex, DocumentId};
|
||||
use group_by::GroupBy;
|
||||
use group_by::{GroupBy, GroupByMut};
|
||||
|
||||
const MAX_DISTANCE: usize = 8;
|
||||
const MAX_DISTANCE: u32 = 8;
|
||||
|
||||
#[derive(Debug, Eq, Clone)]
|
||||
#[inline]
|
||||
fn match_query_index(a: &Match, b: &Match) -> bool {
|
||||
a.query_index == b.query_index
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Document {
|
||||
document_id: DocumentId,
|
||||
matches: Vec<Match>,
|
||||
|
|
@ -33,172 +38,83 @@ impl Document {
|
|||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Document {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.cmp(other) == Ordering::Equal
|
||||
fn sum_of_typos(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
fn sum_of_typos(doc: &Document) -> u8 {
|
||||
GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].distance).sum()
|
||||
}
|
||||
sum_of_typos(lhs).cmp(&sum_of_typos(rhs))
|
||||
}
|
||||
|
||||
impl PartialOrd for Document {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
fn number_of_words(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
fn number_of_words(doc: &Document) -> usize {
|
||||
GroupBy::new(&doc.matches, match_query_index).count()
|
||||
}
|
||||
number_of_words(lhs).cmp(&number_of_words(rhs)).reverse()
|
||||
}
|
||||
|
||||
impl Ord for Document {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
let lhs = DocumentScore::new(&self.matches);
|
||||
let rhs = DocumentScore::new(&other.matches);
|
||||
lhs.cmp(&rhs)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Eq, PartialEq, PartialOrd)]
|
||||
struct DocumentScore {
|
||||
typo: usize,
|
||||
words: usize,
|
||||
proximity: usize,
|
||||
attribute: usize,
|
||||
words_position: usize,
|
||||
}
|
||||
|
||||
impl Ord for DocumentScore {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.typo.cmp(&other.typo)
|
||||
.then(self.words.cmp(&other.words).reverse())
|
||||
.then(self.proximity.cmp(&other.proximity))
|
||||
.then(self.attribute.cmp(&other.attribute))
|
||||
.then(self.words_position.cmp(&other.words_position))
|
||||
// ~exact~ (see prefix option of the `DFA` builder)
|
||||
}
|
||||
}
|
||||
|
||||
fn min_attribute(matches: &[Match]) -> usize {
|
||||
let mut attribute = usize::max_value();
|
||||
for match_ in matches {
|
||||
if match_.attribute == 0 { return 0 }
|
||||
attribute = cmp::min(match_.attribute as usize, attribute);
|
||||
}
|
||||
attribute
|
||||
}
|
||||
|
||||
fn min_attribute_index(matches: &[Match]) -> usize {
|
||||
let mut attribute_index = usize::max_value();
|
||||
for match_ in matches {
|
||||
if match_.attribute_index == 0 { return 0 }
|
||||
attribute_index = cmp::min(match_.attribute_index as usize, attribute_index);
|
||||
}
|
||||
attribute_index
|
||||
}
|
||||
|
||||
impl DocumentScore {
|
||||
fn new(matches: &[Match]) -> Self {
|
||||
let mut score = DocumentScore::default();
|
||||
|
||||
let mut index = 0; // FIXME could be replaced by the `GroupBy::remaining` method
|
||||
for group in GroupBy::new(matches, |a, b| a.query_index == b.query_index) {
|
||||
index += group.len();
|
||||
|
||||
score.typo = cmp::max(group[0].distance as usize, score.typo);
|
||||
score.words += 1;
|
||||
|
||||
// FIXME distance is wrong if 2 different attributes matches
|
||||
if let Some(first_next_group) = (&matches[index..]).first() {
|
||||
score.proximity += attribute_proximity(first_next_group, &group[0]);
|
||||
fn words_proximity(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
fn word_proximity(doc: &Document) -> u32 {
|
||||
fn attribute_proximity(lhs: &Match, rhs: &Match) -> u32 {
|
||||
fn index_proximity(lhs: u32, rhs: u32) -> u32 {
|
||||
if lhs < rhs {
|
||||
cmp::min(rhs - lhs, MAX_DISTANCE)
|
||||
} else {
|
||||
cmp::min(lhs - rhs, MAX_DISTANCE) + 1
|
||||
}
|
||||
}
|
||||
|
||||
score.attribute += min_attribute(group);
|
||||
score.words_position += min_attribute_index(group);
|
||||
if lhs.attribute != rhs.attribute { return MAX_DISTANCE }
|
||||
index_proximity(lhs.attribute_index, rhs.attribute_index)
|
||||
}
|
||||
|
||||
score
|
||||
let mut proximity = 0;
|
||||
let mut next_group_index = 0;
|
||||
for group in GroupBy::new(&doc.matches, match_query_index) {
|
||||
next_group_index += group.len();
|
||||
// FIXME distance is wrong if 2 different attributes matches
|
||||
// FIXME do that in a manner to avoid memory cache misses
|
||||
if let Some(first_next_group) = doc.matches.get(next_group_index) {
|
||||
proximity += attribute_proximity(first_next_group, &group[0]);
|
||||
}
|
||||
}
|
||||
proximity
|
||||
}
|
||||
word_proximity(lhs).cmp(&word_proximity(rhs))
|
||||
}
|
||||
|
||||
fn proximity(first: usize, second: usize) -> usize {
|
||||
if first < second {
|
||||
cmp::min(second - first, MAX_DISTANCE)
|
||||
} else {
|
||||
cmp::min(first - second, MAX_DISTANCE) + 1
|
||||
fn sum_of_words_attribute(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
fn sum_attribute(doc: &Document) -> u8 {
|
||||
GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].attribute).sum()
|
||||
}
|
||||
sum_attribute(lhs).cmp(&sum_attribute(rhs))
|
||||
}
|
||||
|
||||
fn attribute_proximity(lhs: &Match, rhs: &Match) -> usize {
|
||||
if lhs.attribute != rhs.attribute {
|
||||
MAX_DISTANCE
|
||||
} else {
|
||||
let lhs_attr = lhs.attribute_index as usize;
|
||||
let rhs_attr = rhs.attribute_index as usize;
|
||||
proximity(lhs_attr, rhs_attr)
|
||||
fn sum_of_words_position(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
fn sum_attribute_index(doc: &Document) -> u32 {
|
||||
GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].attribute_index).sum()
|
||||
}
|
||||
sum_attribute_index(lhs).cmp(&sum_attribute_index(rhs))
|
||||
}
|
||||
|
||||
fn exact(lhs: &Document, rhs: &Document) -> Ordering {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
pub struct Pool {
|
||||
returned_documents: HashSet<DocumentId>,
|
||||
documents: Vec<Document>,
|
||||
limitation: Limitation,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
enum Limitation {
|
||||
/// No limitation is specified.
|
||||
Unspecified { // FIXME rename that !
|
||||
/// The maximum number of results to return.
|
||||
limit: usize,
|
||||
},
|
||||
|
||||
/// The limitation is specified but not reached.
|
||||
Specified {
|
||||
/// The maximum number of results to return.
|
||||
limit: usize,
|
||||
|
||||
/// documents with a distance of zero which can be used
|
||||
/// in the step-by-step sort-and-return.
|
||||
///
|
||||
/// this field must be equal to the limit to reach
|
||||
/// the limitation
|
||||
matching_documents: usize,
|
||||
},
|
||||
|
||||
/// No more documents with a distance of zero
|
||||
/// can never be returned now.
|
||||
Reached {
|
||||
/// The number of remaining documents to return in order.
|
||||
remaining: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl Limitation {
|
||||
fn reached(&self) -> Option<usize> {
|
||||
match self {
|
||||
Limitation::Reached { remaining } => Some(*remaining),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_reached(&self) -> bool {
|
||||
self.reached().is_some()
|
||||
}
|
||||
limit: usize,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
pub fn new(query_size: usize, limit: usize) -> Self {
|
||||
assert!(query_size > 0, "query size can not be less that one");
|
||||
assert!(limit > 0, "limit can not be less that one");
|
||||
|
||||
let limitation = match query_size {
|
||||
1 => Limitation::Specified { limit, matching_documents: 0 },
|
||||
_ => Limitation::Unspecified { limit },
|
||||
};
|
||||
|
||||
Self {
|
||||
returned_documents: HashSet::new(),
|
||||
documents: Vec::new(),
|
||||
limitation: limitation,
|
||||
limit: limit,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extend(&mut self, mut matches: HashMap<DocumentId, Vec<Match>>) {
|
||||
// TODO remove the matches HashMap, not proud of it
|
||||
pub fn extend(&mut self, matches: &mut HashMap<DocumentId, Vec<Match>>) {
|
||||
for doc in self.documents.iter_mut() {
|
||||
if let Some(matches) = matches.remove(&doc.document_id) {
|
||||
doc.matches.extend(matches);
|
||||
|
|
@ -206,65 +122,48 @@ impl Pool {
|
|||
}
|
||||
}
|
||||
|
||||
matches.retain(|id, _| !self.returned_documents.contains(id));
|
||||
self.documents.reserve(matches.len());
|
||||
|
||||
let mut new_matches = 0;
|
||||
for (id, mut matches) in matches.into_iter() {
|
||||
for (id, mut matches) in matches.drain() {
|
||||
// note that matches are already sorted we do that by security
|
||||
// TODO remove this useless sort
|
||||
matches.sort_unstable();
|
||||
if matches[0].distance == 0 { new_matches += 1 }
|
||||
|
||||
if self.limitation.is_reached() {
|
||||
match matches.iter().position(|match_| match_.distance > 0) {
|
||||
Some(pos) if pos == 0 => continue,
|
||||
Some(pos) => matches.truncate(pos),
|
||||
None => (),
|
||||
}
|
||||
}
|
||||
|
||||
let document = Document::from_sorted_matches(id, matches);
|
||||
self.documents.push(document);
|
||||
}
|
||||
self.documents.sort_unstable();
|
||||
|
||||
self.limitation = match self.limitation {
|
||||
Limitation::Specified { limit, matching_documents } if matching_documents + new_matches >= limit => {
|
||||
// this is the biggest valid match
|
||||
// used to find the next smallest invalid match
|
||||
let biggest_valid = Match { query_index: 0, distance: 0, ..Match::max() };
|
||||
|
||||
// documents which does not have a match with a distance of 0 can be removed.
|
||||
// note that documents have a query size of 1.
|
||||
match self.documents.binary_search_by(|d| d.matches[0].cmp(&biggest_valid)) {
|
||||
Ok(index) => self.documents.truncate(index + 1), // this will never happen :)
|
||||
Err(index) => self.documents.truncate(index),
|
||||
}
|
||||
|
||||
Limitation::Reached { remaining: limit }
|
||||
},
|
||||
Limitation::Specified { limit, matching_documents } => {
|
||||
Limitation::Specified {
|
||||
limit: limit,
|
||||
matching_documents: matching_documents + new_matches
|
||||
}
|
||||
},
|
||||
limitation => limitation,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn invert_sorts<F>(a: &Document, b: &Document, sorts: &[F]) -> bool
|
||||
where F: Fn(&Document, &Document) -> Ordering,
|
||||
{
|
||||
sorts.iter().rev().all(|sort| sort(a, b) == Ordering::Equal)
|
||||
}
|
||||
|
||||
impl IntoIterator for Pool {
|
||||
type Item = Document;
|
||||
type IntoIter = vec::IntoIter<Self::Item>;
|
||||
|
||||
fn into_iter(mut self) -> Self::IntoIter {
|
||||
let limit = match self.limitation {
|
||||
Limitation::Unspecified { limit } => limit,
|
||||
Limitation::Specified { limit, .. } => limit,
|
||||
Limitation::Reached { remaining } => remaining,
|
||||
};
|
||||
let sorts = &[
|
||||
sum_of_typos,
|
||||
number_of_words,
|
||||
words_proximity,
|
||||
sum_of_words_attribute,
|
||||
sum_of_words_position,
|
||||
];
|
||||
|
||||
self.documents.truncate(limit);
|
||||
for (i, sort) in sorts.iter().enumerate() {
|
||||
let mut computed = 0;
|
||||
for group in GroupByMut::new(&mut self.documents, |a, b| invert_sorts(a, b, &sorts[..i])) {
|
||||
// TODO prefer using `sort_unstable_by_key` to allow reusing the key computation
|
||||
// `number of words` needs to be reversed, we can use the `cmp::Reverse` struct to do that
|
||||
group.sort_unstable_by(sort);
|
||||
computed += group.len();
|
||||
if computed >= self.limit { break }
|
||||
}
|
||||
}
|
||||
|
||||
self.documents.truncate(self.limit);
|
||||
self.documents.into_iter()
|
||||
}
|
||||
}
|
||||
|
|
@ -303,6 +202,8 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> {
|
|||
type Item = DocumentId;
|
||||
|
||||
fn next(&'a mut self) -> Option<Self::Item> {
|
||||
let mut matches = HashMap::new();
|
||||
|
||||
loop {
|
||||
// TODO remove that when NLL are here !
|
||||
let mut transfert_pool = None;
|
||||
|
|
@ -313,14 +214,21 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> {
|
|||
Some((_string, indexed_values)) => {
|
||||
for iv in indexed_values {
|
||||
|
||||
// TODO extend documents matches by batch of query_index
|
||||
// that way it will be possible to discard matches that
|
||||
// have an invalid distance *before* adding them
|
||||
// to the matches of the documents and, that way, avoid a sort
|
||||
|
||||
// let string = unsafe { str::from_utf8_unchecked(_string) };
|
||||
// println!("for {:15} ", string);
|
||||
|
||||
let distance = automatons[iv.index].distance(iv.state).to_u8();
|
||||
|
||||
// TODO remove the Pool system !
|
||||
// this is an internal Pool rule but
|
||||
// it is more efficient to test that here
|
||||
if pool.limitation.is_reached() && distance != 0 { continue }
|
||||
// if pool.limitation.is_reached() && distance != 0 { continue }
|
||||
|
||||
let mut matches = HashMap::with_capacity(iv.values.len() / 2);
|
||||
for di in iv.values {
|
||||
let match_ = Match {
|
||||
query_index: iv.index as u32,
|
||||
|
|
@ -329,10 +237,10 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> {
|
|||
attribute_index: di.attribute_index,
|
||||
};
|
||||
matches.entry(di.document)
|
||||
.and_modify(|matches: &mut Vec<_>| matches.push(match_))
|
||||
.and_modify(|ms: &mut Vec<_>| ms.push(match_))
|
||||
.or_insert_with(|| vec![match_]);
|
||||
}
|
||||
pool.extend(matches);
|
||||
pool.extend(&mut matches);
|
||||
}
|
||||
},
|
||||
None => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue