searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Cargo fmt pass

Browse source
This commit is contained in:
Clément Renault 2019-10-18 13:05:28 +02:00
parent 47d777c8f7
commit ca26a0f2e4
No known key found for this signature in database
GPG key ID: 92ADA4E935E71FA4
48 changed files with 1599 additions and 979 deletions
Download patch file Download diff file Expand all files Collapse all files

19
meilidb-core/src/store/docs_words.rs
View file

@ -1,8 +1,8 @@
use std::sync::Arc;
use zlmdb::types::{OwnedType, ByteSlice};
use zlmdb::Result as ZResult;
use crate::DocumentId;
use super::BEU64;
use crate::DocumentId;
use std::sync::Arc;
use zlmdb::types::{ByteSlice, OwnedType};
use zlmdb::Result as ZResult;
#[derive(Copy, Clone)]
pub struct DocsWords {
@ -15,8 +15,7 @@ impl DocsWords {
writer: &mut zlmdb::RwTxn,
document_id: DocumentId,
words: &fst::Set,
) -> ZResult<()>
{
) -> ZResult<()> {
let document_id = BEU64::new(document_id.0);
let bytes = words.as_fst().as_bytes();
self.docs_words.put(writer, &document_id, bytes)
@ -26,8 +25,7 @@ impl DocsWords {
&self,
writer: &mut zlmdb::RwTxn,
document_id: DocumentId,
) -> ZResult<bool>
{
) -> ZResult<bool> {
let document_id = BEU64::new(document_id.0);
self.docs_words.delete(writer, &document_id)
}
@ -36,8 +34,7 @@ impl DocsWords {
&self,
reader: &zlmdb::RoTxn,
document_id: DocumentId,
) -> ZResult<Option<fst::Set>>
{
) -> ZResult<Option<fst::Set>> {
let document_id = BEU64::new(document_id.0);
match self.docs_words.get(reader, &document_id)? {
Some(bytes) => {
@ -45,7 +42,7 @@ impl DocsWords {
let bytes = Arc::from(bytes);
let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
Ok(Some(fst::Set::from(fst)))
},
}
None => Ok(None),
}
}

18
meilidb-core/src/store/documents_fields.rs
View file

@ -1,9 +1,9 @@
use meilidb_schema::SchemaAttr;
use zlmdb::types::{OwnedType, ByteSlice};
use zlmdb::types::{ByteSlice, OwnedType};
use zlmdb::Result as ZResult;
use crate::DocumentId;
use super::DocumentAttrKey;
use crate::DocumentId;
#[derive(Copy, Clone)]
pub struct DocumentsFields {
@ -17,8 +17,7 @@ impl DocumentsFields {
document_id: DocumentId,
attribute: SchemaAttr,
value: &[u8],
) -> ZResult<()>
{
) -> ZResult<()> {
let key = DocumentAttrKey::new(document_id, attribute);
self.documents_fields.put(writer, &key, value)
}
@ -27,8 +26,7 @@ impl DocumentsFields {
&self,
writer: &mut zlmdb::RwTxn,
document_id: DocumentId,
) -> ZResult<usize>
{
) -> ZResult<usize> {
let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
self.documents_fields.delete_range(writer, start..=end)
@ -39,8 +37,7 @@ impl DocumentsFields {
reader: &'txn zlmdb::RoTxn,
document_id: DocumentId,
attribute: SchemaAttr,
) -> ZResult<Option<&'txn [u8]>>
{
) -> ZResult<Option<&'txn [u8]>> {
let key = DocumentAttrKey::new(document_id, attribute);
self.documents_fields.get(reader, &key)
}
@ -49,8 +46,7 @@ impl DocumentsFields {
&self,
reader: &'txn zlmdb::RoTxn,
document_id: DocumentId,
) -> ZResult<DocumentFieldsIter<'txn>>
{
) -> ZResult<DocumentFieldsIter<'txn>> {
let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
let iter = self.documents_fields.range(reader, start..=end)?;
@ -70,7 +66,7 @@ impl<'txn> Iterator for DocumentFieldsIter<'txn> {
Some(Ok((key, bytes))) => {
let attr = SchemaAttr(key.attr.get());
Some(Ok((attr, bytes)))
},
}
Some(Err(e)) => Some(Err(e.into())),
None => None,
}

38
meilidb-core/src/store/documents_fields_counts.rs
View file

@ -1,8 +1,8 @@
use super::DocumentAttrKey;
use crate::DocumentId;
use meilidb_schema::SchemaAttr;
use zlmdb::types::OwnedType;
use zlmdb::Result as ZResult;
use crate::DocumentId;
use super::DocumentAttrKey;
#[derive(Copy, Clone)]
pub struct DocumentsFieldsCounts {
@ -16,8 +16,7 @@ impl DocumentsFieldsCounts {
document_id: DocumentId,
attribute: SchemaAttr,
value: u64,
) -> ZResult<()>
{
) -> ZResult<()> {
let key = DocumentAttrKey::new(document_id, attribute);
self.documents_fields_counts.put(writer, &key, &value)
}
@ -26,11 +25,11 @@ impl DocumentsFieldsCounts {
&self,
writer: &mut zlmdb::RwTxn,
document_id: DocumentId,
) -> ZResult<usize>
{
) -> ZResult<usize> {
let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
self.documents_fields_counts.delete_range(writer, start..=end)
self.documents_fields_counts
.delete_range(writer, start..=end)
}
pub fn document_field_count(
@ -38,8 +37,7 @@ impl DocumentsFieldsCounts {
reader: &zlmdb::RoTxn,
document_id: DocumentId,
attribute: SchemaAttr,
) -> ZResult<Option<u64>>
{
) -> ZResult<Option<u64>> {
let key = DocumentAttrKey::new(document_id, attribute);
match self.documents_fields_counts.get(reader, &key)? {
Some(count) => Ok(Some(count)),
@ -51,8 +49,7 @@ impl DocumentsFieldsCounts {
&self,
reader: &'txn zlmdb::RoTxn,
document_id: DocumentId,
) -> ZResult<DocumentFieldsCountsIter<'txn>>
{
) -> ZResult<DocumentFieldsCountsIter<'txn>> {
let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
let iter = self.documents_fields_counts.range(reader, start..=end)?;
@ -62,17 +59,18 @@ impl DocumentsFieldsCounts {
pub fn documents_ids<'txn>(
&self,
reader: &'txn zlmdb::RoTxn,
) -> ZResult<DocumentsIdsIter<'txn>>
{
) -> ZResult<DocumentsIdsIter<'txn>> {
let iter = self.documents_fields_counts.iter(reader)?;
Ok(DocumentsIdsIter { last_seen_id: None, iter })
Ok(DocumentsIdsIter {
last_seen_id: None,
iter,
})
}
pub fn all_documents_fields_counts<'txn>(
&self,
reader: &'txn zlmdb::RoTxn,
) -> ZResult<AllDocumentsFieldsCountsIter<'txn>>
{
) -> ZResult<AllDocumentsFieldsCountsIter<'txn>> {
let iter = self.documents_fields_counts.iter(reader)?;
Ok(AllDocumentsFieldsCountsIter { iter })
}
@ -90,7 +88,7 @@ impl Iterator for DocumentFieldsCountsIter<'_> {
Some(Ok((key, count))) => {
let attr = SchemaAttr(key.attr.get());
Some(Ok((attr, count)))
},
}
Some(Err(e)) => Some(Err(e.into())),
None => None,
}
@ -112,9 +110,9 @@ impl Iterator for DocumentsIdsIter<'_> {
let document_id = DocumentId(key.docid.get());
if Some(document_id) != self.last_seen_id {
self.last_seen_id = Some(document_id);
return Some(Ok(document_id))
return Some(Ok(document_id));
}
},
}
Err(e) => return Some(Err(e.into())),
}
}
@ -135,7 +133,7 @@ impl<'r> Iterator for AllDocumentsFieldsCountsIter<'r> {
let docid = DocumentId(key.docid.get());
let attr = SchemaAttr(key.attr.get());
Some(Ok((docid, attr, count)))
},
}
Some(Err(e)) => Some(Err(e.into())),
None => None,
}

45
meilidb-core/src/store/main.rs
View file

@ -1,15 +1,15 @@
use std::sync::Arc;
use meilidb_schema::Schema;
use zlmdb::types::{Str, OwnedType, ByteSlice, Serde};
use zlmdb::Result as ZResult;
use crate::RankedMap;
use meilidb_schema::Schema;
use std::sync::Arc;
use zlmdb::types::{ByteSlice, OwnedType, Serde, Str};
use zlmdb::Result as ZResult;
const CUSTOMS_KEY: &str = "customs-key";
const CUSTOMS_KEY: &str = "customs-key";
const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
const RANKED_MAP_KEY: &str = "ranked-map";
const SCHEMA_KEY: &str = "schema";
const SYNONYMS_KEY: &str = "synonyms";
const WORDS_KEY: &str = "words";
const RANKED_MAP_KEY: &str = "ranked-map";
const SCHEMA_KEY: &str = "schema";
const SYNONYMS_KEY: &str = "synonyms";
const WORDS_KEY: &str = "words";
#[derive(Copy, Clone)]
pub struct Main {
@ -29,13 +29,14 @@ impl Main {
let bytes = Arc::from(bytes);
let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
Ok(Some(fst::Set::from(fst)))
},
}
None => Ok(None),
}
}
pub fn put_schema(&self, writer: &mut zlmdb::RwTxn, schema: &Schema) -> ZResult<()> {
self.main.put::<Str, Serde<Schema>>(writer, SCHEMA_KEY, schema)
self.main
.put::<Str, Serde<Schema>>(writer, SCHEMA_KEY, schema)
}
pub fn schema(&self, reader: &zlmdb::RoTxn) -> ZResult<Option<Schema>> {
@ -43,11 +44,13 @@ impl Main {
}
pub fn put_ranked_map(&self, writer: &mut zlmdb::RwTxn, ranked_map: &RankedMap) -> ZResult<()> {
self.main.put::<Str, Serde<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
self.main
.put::<Str, Serde<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
}
pub fn ranked_map(&self, reader: &zlmdb::RoTxn) -> ZResult<Option<RankedMap>> {
self.main.get::<Str, Serde<RankedMap>>(reader, RANKED_MAP_KEY)
self.main
.get::<Str, Serde<RankedMap>>(reader, RANKED_MAP_KEY)
}
pub fn put_synonyms_fst(&self, writer: &mut zlmdb::RwTxn, fst: &fst::Set) -> ZResult<()> {
@ -62,28 +65,34 @@ impl Main {
let bytes = Arc::from(bytes);
let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
Ok(Some(fst::Set::from(fst)))
},
}
None => Ok(None),
}
}
pub fn put_number_of_documents<F>(&self, writer: &mut zlmdb::RwTxn, f: F) -> ZResult<u64>
where F: Fn(u64) -> u64,
where
F: Fn(u64) -> u64,
{
let new = self.number_of_documents(writer).map(f)?;
self.main.put::<Str, OwnedType<u64>>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?;
self.main
.put::<Str, OwnedType<u64>>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?;
Ok(new)
}
pub fn number_of_documents(&self, reader: &zlmdb::RoTxn) -> ZResult<u64> {
match self.main.get::<Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)? {
match self
.main
.get::<Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
{
Some(value) => Ok(value),
None => Ok(0),
}
}
pub fn put_customs(&self, writer: &mut zlmdb::RwTxn, customs: &[u8]) -> ZResult<()> {
self.main.put::<Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
self.main
.put::<Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
}
pub fn customs<'txn>(&self, reader: &'txn zlmdb::RoTxn) -> ZResult<Option<&'txn [u8]>> {

63
meilidb-core/src/store/mod.rs
View file

@ -8,8 +8,10 @@ mod updates;
mod updates_results;
pub use self::docs_words::DocsWords;
pub use self::documents_fields::{DocumentsFields, DocumentFieldsIter};
pub use self::documents_fields_counts::{DocumentsFieldsCounts, DocumentFieldsCountsIter, DocumentsIdsIter};
pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
pub use self::documents_fields_counts::{
DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
};
pub use self::main::Main;
pub use self::postings_lists::PostingsLists;
pub use self::synonyms::Synonyms;
@ -25,19 +27,24 @@ use zlmdb::Result as ZResult;
use crate::criterion::Criteria;
use crate::serde::Deserializer;
use crate::{update, query_builder::QueryBuilder, DocumentId, MResult, Error};
use crate::{query_builder::QueryBuilder, update, DocumentId, Error, MResult};
type BEU64 = zerocopy::U64<byteorder::BigEndian>;
type BEU16 = zerocopy::U16<byteorder::BigEndian>;
#[derive(Debug, Copy, Clone)]
#[derive(AsBytes, FromBytes)]
#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
#[repr(C)]
pub struct DocumentAttrKey { docid: BEU64, attr: BEU16 }
pub struct DocumentAttrKey {
docid: BEU64,
attr: BEU16,
}
impl DocumentAttrKey {
fn new(docid: DocumentId, attr: SchemaAttr) -> DocumentAttrKey {
DocumentAttrKey { docid: BEU64::new(docid.0), attr: BEU16::new(attr.0) }
DocumentAttrKey {
docid: BEU64::new(docid.0),
attr: BEU16::new(attr.0),
}
}
}
@ -93,13 +100,15 @@ impl Index {
reader: &zlmdb::RoTxn,
attributes: Option<&HashSet<&str>>,
document_id: DocumentId,
) -> MResult<Option<T>>
{
) -> MResult<Option<T>> {
let schema = self.main.schema(reader)?;
let schema = schema.ok_or(Error::SchemaMissing)?;
let attributes = match attributes {
Some(attributes) => attributes.into_iter().map(|name| schema.attribute(name)).collect(),
Some(attributes) => attributes
.into_iter()
.map(|name| schema.attribute(name))
.collect(),
None => None,
};
@ -121,9 +130,10 @@ impl Index {
reader: &zlmdb::RoTxn,
document_id: DocumentId,
attribute: SchemaAttr,
) -> MResult<Option<T>>
{
let bytes = self.documents_fields.document_attribute(reader, document_id, attribute)?;
) -> MResult<Option<T>> {
let bytes = self
.documents_fields
.document_attribute(reader, document_id, attribute)?;
match bytes {
Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
None => Ok(None),
@ -183,14 +193,8 @@ impl Index {
&self,
reader: &zlmdb::RoTxn,
update_id: u64,
) -> MResult<update::UpdateStatus>
{
update::update_status(
reader,
self.updates,
self.updates_results,
update_id,
)
) -> MResult<update::UpdateStatus> {
update::update_status(reader, self.updates, self.updates_results, update_id)
}
pub fn query_builder(&self) -> QueryBuilder {
@ -205,8 +209,7 @@ impl Index {
pub fn query_builder_with_criteria<'c, 'f, 'd>(
&self,
criteria: Criteria<'c>,
) -> QueryBuilder<'c, 'f, 'd>
{
) -> QueryBuilder<'c, 'f, 'd> {
QueryBuilder::with_criteria(
self.main,
self.postings_lists,
@ -221,8 +224,7 @@ pub fn create(
env: &zlmdb::Env,
name: &str,
updates_notifier: crossbeam_channel::Sender<()>,
) -> MResult<Index>
{
) -> MResult<Index> {
// create all the store names
let main_name = main_name(name);
let postings_lists_name = postings_lists_name(name);
@ -247,7 +249,9 @@ pub fn create(
main: Main { main },
postings_lists: PostingsLists { postings_lists },
documents_fields: DocumentsFields { documents_fields },
documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
documents_fields_counts: DocumentsFieldsCounts {
documents_fields_counts,
},
synonyms: Synonyms { synonyms },
docs_words: DocsWords { docs_words },
updates: Updates { updates },
@ -260,8 +264,7 @@ pub fn open(
env: &zlmdb::Env,
name: &str,
updates_notifier: crossbeam_channel::Sender<()>,
) -> MResult<Option<Index>>
{
) -> MResult<Option<Index>> {
// create all the store names
let main_name = main_name(name);
let postings_lists_name = postings_lists_name(name);
@ -310,7 +313,9 @@ pub fn open(
main: Main { main },
postings_lists: PostingsLists { postings_lists },
documents_fields: DocumentsFields { documents_fields },
documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
documents_fields_counts: DocumentsFieldsCounts {
documents_fields_counts,
},
synonyms: Synonyms { synonyms },
docs_words: DocsWords { docs_words },
updates: Updates { updates },

10
meilidb-core/src/store/postings_lists.rs
View file

@ -1,8 +1,8 @@
use std::borrow::Cow;
use crate::DocIndex;
use sdset::{Set, SetBuf};
use std::borrow::Cow;
use zlmdb::types::{ByteSlice, CowSlice};
use zlmdb::Result as ZResult;
use crate::DocIndex;
#[derive(Copy, Clone)]
pub struct PostingsLists {
@ -15,8 +15,7 @@ impl PostingsLists {
writer: &mut zlmdb::RwTxn,
word: &[u8],
words_indexes: &Set<DocIndex>,
) -> ZResult<()>
{
) -> ZResult<()> {
self.postings_lists.put(writer, word, words_indexes)
}
@ -28,8 +27,7 @@ impl PostingsLists {
&self,
reader: &'txn zlmdb::RoTxn,
word: &[u8],
) -> ZResult<Option<Cow<'txn, Set<DocIndex>>>>
{
) -> ZResult<Option<Cow<'txn, Set<DocIndex>>>> {
match self.postings_lists.get(reader, word)? {
Some(Cow::Borrowed(slice)) => Ok(Some(Cow::Borrowed(Set::new_unchecked(slice)))),
Some(Cow::Owned(vec)) => Ok(Some(Cow::Owned(SetBuf::new_unchecked(vec)))),

5
meilidb-core/src/store/synonyms.rs
View file

@ -13,8 +13,7 @@ impl Synonyms {
writer: &mut zlmdb::RwTxn,
word: &[u8],
synonyms: &fst::Set,
) -> ZResult<()>
{
) -> ZResult<()> {
let bytes = synonyms.as_fst().as_bytes();
self.synonyms.put(writer, word, bytes)
}
@ -30,7 +29,7 @@ impl Synonyms {
let bytes = Arc::from(bytes);
let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
Ok(Some(fst::Set::from(fst)))
},
}
None => Ok(None),
}
}

25
meilidb-core/src/store/updates.rs
View file

@ -1,13 +1,16 @@
use super::BEU64;
use crate::update::Update;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use zlmdb::types::OwnedType;
use zlmdb::{Result as ZResult, BytesEncode, BytesDecode};
use serde::{Serialize, Deserialize};
use crate::update::Update;
use super::BEU64;
use zlmdb::{BytesDecode, BytesEncode, Result as ZResult};
pub struct SerdeJson<T>(std::marker::PhantomData<T>);
impl<T> BytesEncode for SerdeJson<T> where T: Serialize {
impl<T> BytesEncode for SerdeJson<T>
where
T: Serialize,
{
type EItem = T;
fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
@ -15,7 +18,10 @@ impl<T> BytesEncode for SerdeJson<T> where T: Serialize {
}
}
impl<'a, T: 'a> BytesDecode<'a> for SerdeJson<T> where T: Deserialize<'a> + Clone {
impl<'a, T: 'a> BytesDecode<'a> for SerdeJson<T>
where
T: Deserialize<'a> + Clone,
{
type DItem = T;
fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
@ -56,8 +62,7 @@ impl Updates {
writer: &mut zlmdb::RwTxn,
update_id: u64,
update: &Update,
) -> ZResult<()>
{
) -> ZResult<()> {
// TODO prefer using serde_json?
let update_id = BEU64::new(update_id);
self.updates.put(writer, &update_id, update)
@ -69,8 +74,8 @@ impl Updates {
let key = BEU64::new(update_id);
self.updates.delete(writer, &key)?;
Ok(Some((update_id, update)))
},
None => Ok(None)
}
None => Ok(None),
}
}
}

10
meilidb-core/src/store/updates_results.rs
View file

@ -1,7 +1,7 @@
use super::BEU64;
use crate::update::UpdateResult;
use zlmdb::types::{OwnedType, Serde};
use zlmdb::Result as ZResult;
use crate::update::UpdateResult;
use super::BEU64;
#[derive(Copy, Clone)]
pub struct UpdatesResults {
@ -21,8 +21,7 @@ impl UpdatesResults {
writer: &mut zlmdb::RwTxn,
update_id: u64,
update_result: &UpdateResult,
) -> ZResult<()>
{
) -> ZResult<()> {
let update_id = BEU64::new(update_id);
self.updates_results.put(writer, &update_id, update_result)
}
@ -31,8 +30,7 @@ impl UpdatesResults {
&self,
reader: &zlmdb::RoTxn,
update_id: u64,
) -> ZResult<Option<UpdateResult>>
{
) -> ZResult<Option<UpdateResult>> {
let update_id = BEU64::new(update_id);
self.updates_results.get(reader, &update_id)
}