mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-12-24 05:30:16 +01:00
Truncate facet values that are too long before indexing them
This commit is contained in:
parent
990a861241
commit
ac3baafbe8
@ -6,7 +6,8 @@ use heed::BytesEncode;
|
|||||||
use super::helpers::{create_sorter, sorter_into_reader, try_split_array_at, GrenadParameters};
|
use super::helpers::{create_sorter, sorter_into_reader, try_split_array_at, GrenadParameters};
|
||||||
use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec};
|
use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec};
|
||||||
use crate::heed_codec::StrRefCodec;
|
use crate::heed_codec::StrRefCodec;
|
||||||
use crate::update::index_documents::{merge_cbo_roaring_bitmaps, valid_lmdb_key};
|
use crate::update::index_documents::helpers::MAX_FACET_VALUE_LENGTH;
|
||||||
|
use crate::update::index_documents::merge_cbo_roaring_bitmaps;
|
||||||
use crate::{FieldId, Result};
|
use crate::{FieldId, Result};
|
||||||
|
|
||||||
/// Extracts the facet string and the documents ids where this facet string appear.
|
/// Extracts the facet string and the documents ids where this facet string appear.
|
||||||
@ -38,13 +39,21 @@ pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
|
|||||||
try_split_array_at::<_, 4>(bytes).unwrap();
|
try_split_array_at::<_, 4>(bytes).unwrap();
|
||||||
let document_id = u32::from_be_bytes(document_id_bytes);
|
let document_id = u32::from_be_bytes(document_id_bytes);
|
||||||
|
|
||||||
let normalised_value = std::str::from_utf8(normalized_value_bytes)?;
|
let mut normalised_value = std::str::from_utf8(normalized_value_bytes)?;
|
||||||
|
|
||||||
|
let normalised_truncated_value: String;
|
||||||
|
if normalised_value.len() > MAX_FACET_VALUE_LENGTH {
|
||||||
|
normalised_truncated_value = normalised_value
|
||||||
|
.char_indices()
|
||||||
|
.take_while(|(idx, _)| idx + 4 < MAX_FACET_VALUE_LENGTH)
|
||||||
|
.map(|(_, c)| c)
|
||||||
|
.collect();
|
||||||
|
normalised_value = normalised_truncated_value.as_str();
|
||||||
|
}
|
||||||
let key = FacetGroupKey { field_id, level: 0, left_bound: normalised_value };
|
let key = FacetGroupKey { field_id, level: 0, left_bound: normalised_value };
|
||||||
let key_bytes = FacetGroupKeyCodec::<StrRefCodec>::bytes_encode(&key).unwrap();
|
let key_bytes = FacetGroupKeyCodec::<StrRefCodec>::bytes_encode(&key).unwrap();
|
||||||
if valid_lmdb_key(&key_bytes) {
|
// document id is encoded in native-endian because of the CBO roaring bitmap codec
|
||||||
// document id is encoded in native-endian because of the CBO roaring bitmap codec
|
facet_string_docids_sorter.insert(&key_bytes, document_id.to_ne_bytes())?;
|
||||||
facet_string_docids_sorter.insert(&key_bytes, document_id.to_ne_bytes())?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sorter_into_reader(facet_string_docids_sorter, indexer)
|
sorter_into_reader(facet_string_docids_sorter, indexer)
|
||||||
|
@ -12,6 +12,7 @@ use serde_json::Value;
|
|||||||
use super::helpers::{create_sorter, keep_first, sorter_into_reader, GrenadParameters};
|
use super::helpers::{create_sorter, keep_first, sorter_into_reader, GrenadParameters};
|
||||||
use crate::error::InternalError;
|
use crate::error::InternalError;
|
||||||
use crate::facet::value_encoding::f64_into_bytes;
|
use crate::facet::value_encoding::f64_into_bytes;
|
||||||
|
use crate::update::index_documents::helpers::MAX_FACET_VALUE_LENGTH;
|
||||||
use crate::update::index_documents::{create_writer, writer_into_reader};
|
use crate::update::index_documents::{create_writer, writer_into_reader};
|
||||||
use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32};
|
use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32};
|
||||||
|
|
||||||
@ -85,10 +86,16 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// insert normalized and original facet string in sorter
|
// insert normalized and original facet string in sorter
|
||||||
for (normalized, original) in strings.into_iter().filter(|(n, _)| !n.is_empty()) {
|
for (normalized, original) in strings.into_iter().filter(|(n, _)| !n.is_empty()) {
|
||||||
|
let normalised_truncated_value: String = normalized
|
||||||
|
.char_indices()
|
||||||
|
.take_while(|(idx, _)| idx + 4 < MAX_FACET_VALUE_LENGTH)
|
||||||
|
.map(|(_, c)| c)
|
||||||
|
.collect();
|
||||||
|
|
||||||
key_buffer.truncate(size_of::<FieldId>() + size_of::<DocumentId>());
|
key_buffer.truncate(size_of::<FieldId>() + size_of::<DocumentId>());
|
||||||
key_buffer.extend_from_slice(normalized.as_bytes());
|
key_buffer.extend_from_slice(normalised_truncated_value.as_bytes());
|
||||||
fid_docid_facet_strings_sorter.insert(&key_buffer, original.as_bytes())?;
|
fid_docid_facet_strings_sorter.insert(&key_buffer, original.as_bytes())?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -18,8 +18,20 @@ pub use merge_functions::{
|
|||||||
serialize_roaring_bitmap, MergeFn,
|
serialize_roaring_bitmap, MergeFn,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// The maximum length a LMDB key can be.
|
||||||
|
///
|
||||||
|
/// Note that the actual allowed length is a little bit higher, but
|
||||||
|
/// we keep a margin of safety.
|
||||||
|
const MAX_LMDB_KEY_LENGTH: usize = 500;
|
||||||
|
|
||||||
|
/// The maximum length a field value can be when inserted in an LMDB key.
|
||||||
|
///
|
||||||
|
/// This number is determined by the keys of the different facet databases
|
||||||
|
/// and adding a margin of safety.
|
||||||
|
pub const MAX_FACET_VALUE_LENGTH: usize = MAX_LMDB_KEY_LENGTH - 20;
|
||||||
|
|
||||||
/// The maximum length a word can be
|
/// The maximum length a word can be
|
||||||
pub const MAX_WORD_LENGTH: usize = 250;
|
pub const MAX_WORD_LENGTH: usize = MAX_LMDB_KEY_LENGTH / 2;
|
||||||
|
|
||||||
pub fn valid_lmdb_key(key: impl AsRef<[u8]>) -> bool {
|
pub fn valid_lmdb_key(key: impl AsRef<[u8]>) -> bool {
|
||||||
key.as_ref().len() <= MAX_WORD_LENGTH * 2 && !key.as_ref().is_empty()
|
key.as_ref().len() <= MAX_WORD_LENGTH * 2 && !key.as_ref().is_empty()
|
||||||
|
Loading…
x
Reference in New Issue
Block a user