searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Improve script language codec

Browse Source
This commit is contained in:
f3r10 2022-10-19 07:03:46 -05:00 committed by ManyTheFish
parent fd60a39f1c
commit 2d58b28f43
1 changed files with 10 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
milli/src/heed_codec/script_language_codec.rs
View File

@ -1,6 +1,5 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::mem::size_of;
use std::str; use std::str;
use charabia::{Language, Script}; use charabia::{Language, Script};
@ -11,16 +10,11 @@ impl<'a> heed::BytesDecode<'a> for ScriptLanguageCodec {
type DItem = (Script, Language); type DItem = (Script, Language);
fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> { fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
let footer_len = size_of::<u32>(); let sep = bytes.iter().position(|b| *b == 0)?;
let (s_bytes, l_bytes) = bytes.split_at(sep);
if bytes.len() < footer_len { let script = str::from_utf8(s_bytes).ok()?;
return None;
}
let (script, bytes) = bytes.split_at(bytes.len() - footer_len);
let script = str::from_utf8(script).ok()?;
let script_name = Script::from_name(script); let script_name = Script::from_name(script);
let lan = str::from_utf8(bytes).ok()?; let lan = str::from_utf8(l_bytes).ok()?;
let lan_name = Language::from_name(lan); let lan_name = Language::from_name(lan);
Some((script_name, lan_name)) Some((script_name, lan_name))
@ -31,12 +25,13 @@ impl<'a> heed::BytesEncode<'a> for ScriptLanguageCodec {
type EItem = (Script, Language); type EItem = (Script, Language);
fn bytes_encode((script, lan): &Self::EItem) -> Option<Cow<[u8]>> { fn bytes_encode((script, lan): &Self::EItem) -> Option<Cow<[u8]>> {
let script_name = script.name(); let script_name = script.name().as_bytes();
let lan_name = lan.name(); let lan_name = lan.name().as_bytes();
let mut bytes = Vec::with_capacity(script_name.len() + lan_name.len()); let mut bytes = Vec::with_capacity(script_name.len() + lan_name.len() + 1);
bytes.extend_from_slice(script_name.as_bytes()); bytes.extend_from_slice(script_name);
bytes.extend_from_slice(lan_name.as_bytes()); bytes.push(0);
bytes.extend_from_slice(lan_name);
Some(Cow::Owned(bytes)) Some(Cow::Owned(bytes))
} }