Split the index-scheduler in ~500 loc modules

This commit is contained in:
Tamo 2025-01-02 14:13:51 +01:00
parent 7f1071943e
commit cb82b0798a
No known key found for this signature in database
GPG key ID: 20CD8020AFA88D69
251 changed files with 9431 additions and 9079 deletions

View file

@ -307,7 +307,7 @@ fn open_or_create_database_unchecked(
task_db_size: opt.max_task_db_size.as_u64() as usize,
index_base_map_size: opt.max_index_size.as_u64() as usize,
enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
indexer_config: (&opt.indexer_options).try_into()?,
indexer_config: Arc::new((&opt.indexer_options).try_into()?),
autobatching_enabled: true,
cleanup_enabled: !opt.experimental_replication_parameters,
max_number_of_tasks: 1_000_000,

View file

@ -36,7 +36,7 @@ async fn get_batch(
let query = index_scheduler::Query { batch_uids: Some(vec![batch_uid]), ..Query::default() };
let filters = index_scheduler.filters();
let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(query, filters)?;
let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?;
if let Some(batch) = batches.first() {
let task_view = BatchView::from_batch(batch);
@ -66,7 +66,7 @@ async fn get_batches(
let query = params.into_query();
let filters = index_scheduler.filters();
let (tasks, total) = index_scheduler.get_batches_from_authorized_indexes(query, filters)?;
let (tasks, total) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?;
let mut results: Vec<_> = tasks.iter().map(BatchView::from_batch).collect();
// If we were able to fetch the number +1 tasks we asked

View file

@ -608,7 +608,7 @@ async fn document_addition(
}
};
let (uuid, mut update_file) = index_scheduler.create_update_file(dry_run)?;
let (uuid, mut update_file) = index_scheduler.queue.create_update_file(dry_run)?;
let documents_count = match format {
PayloadType::Ndjson => {
let (path, file) = update_file.into_parts();
@ -670,7 +670,7 @@ async fn document_addition(
Err(e) => {
// Here the file MAY have been persisted or not.
// We don't know thus we ignore the file not found error.
match index_scheduler.delete_update_file(uuid) {
match index_scheduler.queue.delete_update_file(uuid) {
Ok(()) => (),
Err(index_scheduler::Error::FileStore(file_store::Error::IoError(e)))
if e.kind() == ErrorKind::NotFound => {}
@ -701,7 +701,7 @@ async fn document_addition(
{
Ok(task) => task,
Err(e) => {
index_scheduler.delete_update_file(uuid)?;
index_scheduler.queue.delete_update_file(uuid)?;
return Err(e.into());
}
};

View file

@ -1,7 +1,3 @@
use crate::extractors::authentication::policies::ActionPolicy;
use crate::extractors::authentication::{AuthenticationError, GuardedData};
use crate::routes::create_all_stats;
use crate::search_queue::SearchQueue;
use actix_web::http::header;
use actix_web::web::{self, Data};
use actix_web::HttpResponse;
@ -13,6 +9,11 @@ use meilisearch_types::tasks::Status;
use prometheus::{Encoder, TextEncoder};
use time::OffsetDateTime;
use crate::extractors::authentication::policies::ActionPolicy;
use crate::extractors::authentication::{AuthenticationError, GuardedData};
use crate::routes::create_all_stats;
use crate::search_queue::SearchQueue;
pub fn configure(config: &mut web::ServiceConfig) {
config.service(web::resource("").route(web::get().to(get_metrics)));
}
@ -64,7 +65,7 @@ pub async fn get_metrics(
let task_queue_latency_seconds = index_scheduler
.get_tasks_from_authorized_indexes(
Query {
&Query {
limit: Some(1),
reverse: Some(true),
statuses: Some(vec![Status::Enqueued, Status::Processing]),

View file

@ -260,11 +260,8 @@ async fn cancel_tasks(
let query = params.into_query();
let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes(
&index_scheduler.read_txn()?,
&query,
index_scheduler.filters(),
)?;
let (tasks, _) =
index_scheduler.get_task_ids_from_authorized_indexes(&query, index_scheduler.filters())?;
let task_cancelation =
KindWithContent::TaskCancelation { query: format!("?{}", req.query_string()), tasks };
@ -312,11 +309,8 @@ async fn delete_tasks(
let query = params.into_query();
let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes(
&index_scheduler.read_txn()?,
&query,
index_scheduler.filters(),
)?;
let (tasks, _) =
index_scheduler.get_task_ids_from_authorized_indexes(&query, index_scheduler.filters())?;
let task_deletion =
KindWithContent::TaskDeletion { query: format!("?{}", req.query_string()), tasks };
@ -349,7 +343,7 @@ async fn get_tasks(
let query = params.into_query();
let filters = index_scheduler.filters();
let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?;
let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(&query, filters)?;
let mut results: Vec<_> = tasks.iter().map(TaskView::from_task).collect();
// If we were able to fetch the number +1 tasks we asked
@ -377,7 +371,7 @@ async fn get_task(
let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() };
let filters = index_scheduler.filters();
let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?;
let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(&query, filters)?;
if let Some(task) = tasks.first() {
let task_view = TaskView::from_task(task);