1533: Update milli version to v0.8.0 r=MarinPostma a=curquiza

- Update milli, heed and obkv
- fix relevancy issue and the `facetsDistribution` display

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
This commit is contained in:
bors[bot] 2021-07-28 11:15:31 +00:00 committed by GitHub
commit 3e30d4270b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 22 deletions

57
Cargo.lock generated
View File

@ -620,6 +620,17 @@ dependencies = [
"vec_map", "vec_map",
] ]
[[package]]
name = "concat-arrays"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747"
dependencies = [
"proc-macro2 1.0.27",
"quote 1.0.9",
"syn 1.0.73",
]
[[package]] [[package]]
name = "const_fn" name = "const_fn"
version = "0.4.8" version = "0.4.8"
@ -1191,7 +1202,7 @@ dependencies = [
[[package]] [[package]]
name = "heed" name = "heed"
version = "0.12.0" version = "0.12.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"heed-traits", "heed-traits",
@ -1209,12 +1220,12 @@ dependencies = [
[[package]] [[package]]
name = "heed-traits" name = "heed-traits"
version = "0.7.0" version = "0.7.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
[[package]] [[package]]
name = "heed-types" name = "heed-types"
version = "0.7.2" version = "0.7.2"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [ dependencies = [
"bincode", "bincode",
"heed-traits", "heed-traits",
@ -1647,7 +1658,7 @@ dependencies = [
"log", "log",
"main_error", "main_error",
"meilisearch-error", "meilisearch-error",
"meilisearch-tokenizer", "meilisearch-tokenizer 0.2.3",
"memmap", "memmap",
"milli", "milli",
"mime", "mime",
@ -1699,7 +1710,23 @@ dependencies = [
"once_cell", "once_cell",
"slice-group-by", "slice-group-by",
"unicode-segmentation", "unicode-segmentation",
"whatlang", "whatlang 0.9.0",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.4"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang 0.12.0",
] ]
[[package]] [[package]]
@ -1729,12 +1756,13 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.7.2" version = "0.8.0"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.2#007fec21fcb3accad807aa2f2cf66c168afb043f" source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.0#200e98c211510750e2ead475b8571b533ac6caa1"
dependencies = [ dependencies = [
"bstr", "bstr",
"byteorder", "byteorder",
"chrono", "chrono",
"concat-arrays",
"csv", "csv",
"either", "either",
"flate2", "flate2",
@ -1748,7 +1776,7 @@ dependencies = [
"linked-hash-map", "linked-hash-map",
"log", "log",
"logging_timer", "logging_timer",
"meilisearch-tokenizer", "meilisearch-tokenizer 0.2.4",
"memmap", "memmap",
"obkv", "obkv",
"once_cell", "once_cell",
@ -1910,9 +1938,9 @@ dependencies = [
[[package]] [[package]]
name = "obkv" name = "obkv"
version = "0.1.1" version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385"
[[package]] [[package]]
name = "once_cell" name = "once_cell"
@ -3532,6 +3560,15 @@ dependencies = [
"hashbrown 0.7.2", "hashbrown 0.7.2",
] ]
[[package]]
name = "whatlang"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703"
dependencies = [
"hashbrown 0.7.2",
]
[[package]] [[package]]
name = "whoami" name = "whoami"
version = "1.1.2" version = "1.1.2"

View File

@ -42,7 +42,7 @@ fst = "0.4.5"
futures = "0.3.7" futures = "0.3.7"
futures-util = "0.3.8" futures-util = "0.3.8"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
http = "0.2.1" http = "0.2.1"
indexmap = { version = "1.3.2", features = ["serde-1"] } indexmap = { version = "1.3.2", features = ["serde-1"] }
itertools = "0.10.0" itertools = "0.10.0"
@ -51,7 +51,7 @@ main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" } meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
memmap = "0.7.0" memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.2" } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" }
mime = "0.3.16" mime = "0.3.16"
num_cpus = "1.13.0" num_cpus = "1.13.0"
once_cell = "1.5.2" once_cell = "1.5.2"
@ -73,7 +73,7 @@ thiserror = "1.0.24"
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
uuid = { version = "0.8.2", features = ["serde"] } uuid = { version = "0.8.2", features = ["serde"] }
walkdir = "2.3.2" walkdir = "2.3.2"
obkv = "0.1.1" obkv = "0.2.0"
pin-project = "1.0.7" pin-project = "1.0.7"
whoami = { version = "1.1.2", optional = true } whoami = { version = "1.1.2", optional = true }
reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }

View File

@ -6,7 +6,7 @@ use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use heed::{EnvOpenOptions, RoTxn}; use heed::{EnvOpenOptions, RoTxn};
use milli::obkv_to_json; use milli::{obkv_to_json, FieldId};
use serde::{de::Deserializer, Deserialize}; use serde::{de::Deserializer, Deserialize};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
@ -174,7 +174,7 @@ impl Index {
txn: &heed::RoTxn, txn: &heed::RoTxn,
attributes_to_retrieve: &Option<Vec<S>>, attributes_to_retrieve: &Option<Vec<S>>,
fields_ids_map: &milli::FieldsIdsMap, fields_ids_map: &milli::FieldsIdsMap,
) -> Result<Vec<u8>> { ) -> Result<Vec<FieldId>> {
let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? {
Some(ids) => ids.into_iter().collect::<Vec<_>>(), Some(ids) => ids.into_iter().collect::<Vec<_>>(),
None => fields_ids_map.iter().map(|(id, _)| id).collect(), None => fields_ids_map.iter().map(|(id, _)| id).collect(),

View File

@ -281,9 +281,9 @@ fn compute_formatted_options(
attr_to_highlight: &HashSet<String>, attr_to_highlight: &HashSet<String>,
attr_to_crop: &[String], attr_to_crop: &[String],
query_crop_length: usize, query_crop_length: usize,
to_retrieve_ids: &BTreeSet<u8>, to_retrieve_ids: &BTreeSet<FieldId>,
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>, displayed_ids: &BTreeSet<FieldId>,
) -> BTreeMap<FieldId, FormatOptions> { ) -> BTreeMap<FieldId, FormatOptions> {
let mut formatted_options = BTreeMap::new(); let mut formatted_options = BTreeMap::new();
@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options(
formatted_options: &mut BTreeMap<FieldId, FormatOptions>, formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
attr_to_highlight: &HashSet<String>, attr_to_highlight: &HashSet<String>,
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>, displayed_ids: &BTreeSet<FieldId>,
) { ) {
for attr in attr_to_highlight { for attr in attr_to_highlight {
let new_format = FormatOptions { let new_format = FormatOptions {
@ -342,7 +342,7 @@ fn add_crop_to_formatted_options(
attr_to_crop: &[String], attr_to_crop: &[String],
crop_length: usize, crop_length: usize,
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>, displayed_ids: &BTreeSet<FieldId>,
) { ) {
for attr in attr_to_crop { for attr in attr_to_crop {
let mut split = attr.rsplitn(2, ':'); let mut split = attr.rsplitn(2, ':');
@ -382,7 +382,7 @@ fn add_crop_to_formatted_options(
fn add_non_formatted_ids_to_formatted_options( fn add_non_formatted_ids_to_formatted_options(
formatted_options: &mut BTreeMap<FieldId, FormatOptions>, formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
to_retrieve_ids: &BTreeSet<u8>, to_retrieve_ids: &BTreeSet<FieldId>,
) { ) {
for id in to_retrieve_ids { for id in to_retrieve_ids {
formatted_options.entry(*id).or_insert(FormatOptions { formatted_options.entry(*id).or_insert(FormatOptions {
@ -395,7 +395,7 @@ fn add_non_formatted_ids_to_formatted_options(
fn make_document( fn make_document(
attributes_to_retrieve: &BTreeSet<FieldId>, attributes_to_retrieve: &BTreeSet<FieldId>,
field_ids_map: &FieldsIdsMap, field_ids_map: &FieldsIdsMap,
obkv: obkv::KvReader, obkv: obkv::KvReaderU16,
) -> Result<Document> { ) -> Result<Document> {
let mut document = Document::new(); let mut document = Document::new();
@ -418,7 +418,7 @@ fn make_document(
fn format_fields<A: AsRef<[u8]>>( fn format_fields<A: AsRef<[u8]>>(
field_ids_map: &FieldsIdsMap, field_ids_map: &FieldsIdsMap,
obkv: obkv::KvReader, obkv: obkv::KvReaderU16,
formatter: &Formatter<A>, formatter: &Formatter<A>,
matching_words: &impl Matcher, matching_words: &impl Matcher,
formatted_options: &BTreeMap<FieldId, FormatOptions>, formatted_options: &BTreeMap<FieldId, FormatOptions>,