mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-03 03:47:02 +02:00
Split the index-scheduler in ~500 loc modules
This commit is contained in:
parent
7f1071943e
commit
cb82b0798a
251 changed files with 9431 additions and 9079 deletions
|
@ -307,7 +307,7 @@ fn open_or_create_database_unchecked(
|
|||
task_db_size: opt.max_task_db_size.as_u64() as usize,
|
||||
index_base_map_size: opt.max_index_size.as_u64() as usize,
|
||||
enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
|
||||
indexer_config: (&opt.indexer_options).try_into()?,
|
||||
indexer_config: Arc::new((&opt.indexer_options).try_into()?),
|
||||
autobatching_enabled: true,
|
||||
cleanup_enabled: !opt.experimental_replication_parameters,
|
||||
max_number_of_tasks: 1_000_000,
|
||||
|
|
|
@ -36,7 +36,7 @@ async fn get_batch(
|
|||
|
||||
let query = index_scheduler::Query { batch_uids: Some(vec![batch_uid]), ..Query::default() };
|
||||
let filters = index_scheduler.filters();
|
||||
let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(query, filters)?;
|
||||
let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?;
|
||||
|
||||
if let Some(batch) = batches.first() {
|
||||
let task_view = BatchView::from_batch(batch);
|
||||
|
@ -66,7 +66,7 @@ async fn get_batches(
|
|||
let query = params.into_query();
|
||||
|
||||
let filters = index_scheduler.filters();
|
||||
let (tasks, total) = index_scheduler.get_batches_from_authorized_indexes(query, filters)?;
|
||||
let (tasks, total) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?;
|
||||
let mut results: Vec<_> = tasks.iter().map(BatchView::from_batch).collect();
|
||||
|
||||
// If we were able to fetch the number +1 tasks we asked
|
||||
|
|
|
@ -608,7 +608,7 @@ async fn document_addition(
|
|||
}
|
||||
};
|
||||
|
||||
let (uuid, mut update_file) = index_scheduler.create_update_file(dry_run)?;
|
||||
let (uuid, mut update_file) = index_scheduler.queue.create_update_file(dry_run)?;
|
||||
let documents_count = match format {
|
||||
PayloadType::Ndjson => {
|
||||
let (path, file) = update_file.into_parts();
|
||||
|
@ -670,7 +670,7 @@ async fn document_addition(
|
|||
Err(e) => {
|
||||
// Here the file MAY have been persisted or not.
|
||||
// We don't know thus we ignore the file not found error.
|
||||
match index_scheduler.delete_update_file(uuid) {
|
||||
match index_scheduler.queue.delete_update_file(uuid) {
|
||||
Ok(()) => (),
|
||||
Err(index_scheduler::Error::FileStore(file_store::Error::IoError(e)))
|
||||
if e.kind() == ErrorKind::NotFound => {}
|
||||
|
@ -701,7 +701,7 @@ async fn document_addition(
|
|||
{
|
||||
Ok(task) => task,
|
||||
Err(e) => {
|
||||
index_scheduler.delete_update_file(uuid)?;
|
||||
index_scheduler.queue.delete_update_file(uuid)?;
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
|
|
@ -1,7 +1,3 @@
|
|||
use crate::extractors::authentication::policies::ActionPolicy;
|
||||
use crate::extractors::authentication::{AuthenticationError, GuardedData};
|
||||
use crate::routes::create_all_stats;
|
||||
use crate::search_queue::SearchQueue;
|
||||
use actix_web::http::header;
|
||||
use actix_web::web::{self, Data};
|
||||
use actix_web::HttpResponse;
|
||||
|
@ -13,6 +9,11 @@ use meilisearch_types::tasks::Status;
|
|||
use prometheus::{Encoder, TextEncoder};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::extractors::authentication::policies::ActionPolicy;
|
||||
use crate::extractors::authentication::{AuthenticationError, GuardedData};
|
||||
use crate::routes::create_all_stats;
|
||||
use crate::search_queue::SearchQueue;
|
||||
|
||||
pub fn configure(config: &mut web::ServiceConfig) {
|
||||
config.service(web::resource("").route(web::get().to(get_metrics)));
|
||||
}
|
||||
|
@ -64,7 +65,7 @@ pub async fn get_metrics(
|
|||
|
||||
let task_queue_latency_seconds = index_scheduler
|
||||
.get_tasks_from_authorized_indexes(
|
||||
Query {
|
||||
&Query {
|
||||
limit: Some(1),
|
||||
reverse: Some(true),
|
||||
statuses: Some(vec![Status::Enqueued, Status::Processing]),
|
||||
|
|
|
@ -260,11 +260,8 @@ async fn cancel_tasks(
|
|||
|
||||
let query = params.into_query();
|
||||
|
||||
let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes(
|
||||
&index_scheduler.read_txn()?,
|
||||
&query,
|
||||
index_scheduler.filters(),
|
||||
)?;
|
||||
let (tasks, _) =
|
||||
index_scheduler.get_task_ids_from_authorized_indexes(&query, index_scheduler.filters())?;
|
||||
let task_cancelation =
|
||||
KindWithContent::TaskCancelation { query: format!("?{}", req.query_string()), tasks };
|
||||
|
||||
|
@ -312,11 +309,8 @@ async fn delete_tasks(
|
|||
|
||||
let query = params.into_query();
|
||||
|
||||
let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes(
|
||||
&index_scheduler.read_txn()?,
|
||||
&query,
|
||||
index_scheduler.filters(),
|
||||
)?;
|
||||
let (tasks, _) =
|
||||
index_scheduler.get_task_ids_from_authorized_indexes(&query, index_scheduler.filters())?;
|
||||
let task_deletion =
|
||||
KindWithContent::TaskDeletion { query: format!("?{}", req.query_string()), tasks };
|
||||
|
||||
|
@ -349,7 +343,7 @@ async fn get_tasks(
|
|||
let query = params.into_query();
|
||||
|
||||
let filters = index_scheduler.filters();
|
||||
let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?;
|
||||
let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(&query, filters)?;
|
||||
let mut results: Vec<_> = tasks.iter().map(TaskView::from_task).collect();
|
||||
|
||||
// If we were able to fetch the number +1 tasks we asked
|
||||
|
@ -377,7 +371,7 @@ async fn get_task(
|
|||
|
||||
let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() };
|
||||
let filters = index_scheduler.filters();
|
||||
let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?;
|
||||
let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(&query, filters)?;
|
||||
|
||||
if let Some(task) = tasks.first() {
|
||||
let task_view = TaskView::from_task(task);
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue