mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-03 11:57:07 +02:00
first version working with index operation
This commit is contained in:
Tamo
parent
f1aa22567f
commit
301907869d
7 changed files with 161 additions and 74 deletions
|
|
@ -23,6 +23,7 @@ use std::fs::{self, File};
|
|||
use std::io::BufWriter;
|
||||
|
||||
use cluster::Consistency;
|
||||
use crossbeam::utils::Backoff;
|
||||
use dump::IndexMetadata;
|
||||
use log::{debug, error, info};
|
||||
use meilisearch_types::heed::{RoTxn, RwTxn};
|
||||
|
|
@ -588,12 +589,8 @@ impl IndexScheduler {
|
|||
}
|
||||
|
||||
match &self.cluster {
|
||||
Some(Cluster::Leader(leader)) => {
|
||||
leader.write().unwrap().commit(Consistency::All)
|
||||
}
|
||||
Some(Cluster::Follower(follower)) => {
|
||||
follower.write().unwrap().ready_to_commit()
|
||||
}
|
||||
Some(Cluster::Leader(leader)) => leader.commit(Consistency::All),
|
||||
Some(Cluster::Follower(follower)) => follower.ready_to_commit(),
|
||||
None => (),
|
||||
}
|
||||
|
||||
|
|
@ -642,12 +639,8 @@ impl IndexScheduler {
|
|||
}
|
||||
|
||||
match &self.cluster {
|
||||
Some(Cluster::Leader(leader)) => {
|
||||
leader.write().unwrap().commit(Consistency::All)
|
||||
}
|
||||
Some(Cluster::Follower(follower)) => {
|
||||
follower.write().unwrap().ready_to_commit()
|
||||
}
|
||||
Some(Cluster::Leader(leader)) => leader.commit(Consistency::All),
|
||||
Some(Cluster::Follower(follower)) => follower.ready_to_commit(),
|
||||
None => (),
|
||||
}
|
||||
|
||||
|
|
@ -864,12 +857,8 @@ impl IndexScheduler {
|
|||
let tasks = self.apply_index_operation(&mut index_wtxn, &index, op)?;
|
||||
|
||||
match &self.cluster {
|
||||
Some(Cluster::Leader(leader)) => {
|
||||
leader.write().unwrap().commit(Consistency::All)
|
||||
}
|
||||
Some(Cluster::Follower(follower)) => {
|
||||
follower.write().unwrap().ready_to_commit()
|
||||
}
|
||||
Some(Cluster::Leader(leader)) => leader.commit(Consistency::All),
|
||||
Some(Cluster::Follower(follower)) => follower.ready_to_commit(),
|
||||
None => (),
|
||||
}
|
||||
|
||||
|
|
@ -973,12 +962,8 @@ impl IndexScheduler {
|
|||
}
|
||||
|
||||
match &self.cluster {
|
||||
Some(Cluster::Leader(leader)) => {
|
||||
leader.write().unwrap().commit(Consistency::All)
|
||||
}
|
||||
Some(Cluster::Follower(follower)) => {
|
||||
follower.write().unwrap().ready_to_commit()
|
||||
}
|
||||
Some(Cluster::Leader(leader)) => leader.commit(Consistency::All),
|
||||
Some(Cluster::Follower(follower)) => follower.ready_to_commit(),
|
||||
None => (),
|
||||
}
|
||||
|
||||
|
|
@ -1422,51 +1407,69 @@ impl IndexScheduler {
|
|||
|
||||
pub(crate) fn get_batch_from_cluster_batch(
|
||||
&self,
|
||||
rtxn: &RoTxn,
|
||||
batch: cluster::batch::Batch,
|
||||
) -> Result<Batch> {
|
||||
use cluster::batch::Batch as CBatch;
|
||||
|
||||
let mut rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
|
||||
|
||||
for id in batch.ids() {
|
||||
let backoff = Backoff::new();
|
||||
let id = BEU32::new(id);
|
||||
|
||||
loop {
|
||||
if self.all_tasks.get(&rtxn, &id)?.is_some() {
|
||||
info!("Found the task_id");
|
||||
break;
|
||||
}
|
||||
info!("The task is not present in the task queue, we wait");
|
||||
// we need to drop the txn to make a write visible
|
||||
drop(rtxn);
|
||||
backoff.spin();
|
||||
rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(match batch {
|
||||
CBatch::TaskCancelation { task, previous_started_at, previous_processing_tasks } => {
|
||||
Batch::TaskCancelation {
|
||||
task: self.get_existing_tasks(rtxn, Some(task))?[0].clone(),
|
||||
task: self.get_existing_tasks(&rtxn, Some(task))?[0].clone(),
|
||||
previous_started_at,
|
||||
previous_processing_tasks,
|
||||
}
|
||||
}
|
||||
CBatch::TaskDeletion(task) => {
|
||||
Batch::TaskDeletion(self.get_existing_tasks(rtxn, Some(task))?[0].clone())
|
||||
Batch::TaskDeletion(self.get_existing_tasks(&rtxn, Some(task))?[0].clone())
|
||||
}
|
||||
CBatch::SnapshotCreation(tasks) => {
|
||||
Batch::SnapshotCreation(self.get_existing_tasks(rtxn, tasks)?)
|
||||
Batch::SnapshotCreation(self.get_existing_tasks(&rtxn, tasks)?)
|
||||
}
|
||||
CBatch::Dump(task) => {
|
||||
Batch::Dump(self.get_existing_tasks(rtxn, Some(task))?[0].clone())
|
||||
Batch::Dump(self.get_existing_tasks(&rtxn, Some(task))?[0].clone())
|
||||
}
|
||||
CBatch::IndexOperation { op, must_create_index } => Batch::IndexOperation {
|
||||
op: self.get_index_op_from_cluster_index_op(rtxn, op)?,
|
||||
op: self.get_index_op_from_cluster_index_op(&rtxn, op)?,
|
||||
must_create_index,
|
||||
},
|
||||
CBatch::IndexCreation { index_uid, primary_key, task } => Batch::IndexCreation {
|
||||
index_uid,
|
||||
primary_key,
|
||||
task: self.get_existing_tasks(rtxn, Some(task))?[0].clone(),
|
||||
task: self.get_existing_tasks(&rtxn, Some(task))?[0].clone(),
|
||||
},
|
||||
CBatch::IndexUpdate { index_uid, primary_key, task } => Batch::IndexUpdate {
|
||||
index_uid,
|
||||
primary_key,
|
||||
task: self.get_existing_tasks(rtxn, Some(task))?[0].clone(),
|
||||
task: self.get_existing_tasks(&rtxn, Some(task))?[0].clone(),
|
||||
},
|
||||
CBatch::IndexDeletion { index_uid, tasks, index_has_been_created } => {
|
||||
Batch::IndexDeletion {
|
||||
index_uid,
|
||||
tasks: self.get_existing_tasks(rtxn, tasks)?,
|
||||
tasks: self.get_existing_tasks(&rtxn, tasks)?,
|
||||
index_has_been_created,
|
||||
}
|
||||
}
|
||||
CBatch::IndexSwap { task } => {
|
||||
Batch::IndexSwap { task: self.get_existing_tasks(rtxn, Some(task))?[0].clone() }
|
||||
Batch::IndexSwap { task: self.get_existing_tasks(&rtxn, Some(task))?[0].clone() }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -349,8 +349,8 @@ impl std::str::FromStr for ClusterMode {
|
|||
|
||||
#[derive(Clone)]
|
||||
pub enum Cluster {
|
||||
Leader(Arc<RwLock<Leader>>),
|
||||
Follower(Arc<RwLock<Follower>>),
|
||||
Leader(Leader),
|
||||
Follower(Follower),
|
||||
}
|
||||
|
||||
impl IndexScheduler {
|
||||
|
|
@ -375,7 +375,7 @@ impl IndexScheduler {
|
|||
dumps_path: self.dumps_path.clone(),
|
||||
auth_path: self.auth_path.clone(),
|
||||
version_file_path: self.version_file_path.clone(),
|
||||
cluster: None,
|
||||
cluster: self.cluster.clone(),
|
||||
#[cfg(test)]
|
||||
test_breakpoint_sdr: self.test_breakpoint_sdr.clone(),
|
||||
#[cfg(test)]
|
||||
|
|
@ -544,11 +544,22 @@ impl IndexScheduler {
|
|||
fn run(&self) {
|
||||
let run = self.private_clone();
|
||||
|
||||
if run.cluster.is_some() {
|
||||
log::warn!("Run in a cluster");
|
||||
} else {
|
||||
log::warn!("Run not in a cluster");
|
||||
}
|
||||
|
||||
// if we're a follower we starts a thread to register the tasks coming from the leader
|
||||
if let Some(Cluster::Follower(follower)) = self.cluster.clone() {
|
||||
let this = self.private_clone();
|
||||
if this.cluster.is_some() {
|
||||
log::warn!("this in a cluster");
|
||||
} else {
|
||||
log::warn!("this not in a cluster");
|
||||
}
|
||||
std::thread::spawn(move || loop {
|
||||
let (task, content) = follower.write().unwrap().get_new_task();
|
||||
let (task, content) = follower.get_new_task();
|
||||
this.register_raw_task(task, content);
|
||||
});
|
||||
}
|
||||
|
|
@ -917,7 +928,7 @@ impl IndexScheduler {
|
|||
} else {
|
||||
None
|
||||
};
|
||||
leader.write().unwrap().register_new_task(task.clone(), update_file);
|
||||
leader.register_new_task(task.clone(), update_file);
|
||||
}
|
||||
|
||||
// If the registered task is a task cancelation
|
||||
|
|
@ -1283,21 +1294,30 @@ impl IndexScheduler {
|
|||
/// If there is no cluster or if leader -> create a new batch
|
||||
/// If follower -> wait till the leader gives us a batch to process
|
||||
fn get_or_create_next_batch(&self) -> Result<Option<Batch>> {
|
||||
let rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
|
||||
info!("inside get or create next batch");
|
||||
|
||||
let batch = match &self.cluster {
|
||||
None | Some(Cluster::Leader(_)) => {
|
||||
let rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
|
||||
self.create_next_batch(&rtxn).map_err(|e| Error::CreateBatch(Box::new(e)))?
|
||||
}
|
||||
Some(Cluster::Follower(follower)) => {
|
||||
let batch = follower.write().unwrap().get_new_batch();
|
||||
Some(self.get_batch_from_cluster_batch(&rtxn, batch)?)
|
||||
let batch = follower.get_new_batch();
|
||||
Some(self.get_batch_from_cluster_batch(batch)?)
|
||||
}
|
||||
};
|
||||
|
||||
if self.cluster.is_some() {
|
||||
println!("HERE: Part of a cluster");
|
||||
} else if self.cluster.is_none() {
|
||||
println!("HERE: Not part of a cluster");
|
||||
}
|
||||
info!("before checking if i’m a leader");
|
||||
if let Some(Cluster::Leader(leader)) = &self.cluster {
|
||||
info!("I'm a leader");
|
||||
if let Some(ref batch) = batch {
|
||||
leader.write().unwrap().starts_batch(batch.clone().into());
|
||||
info!("I'm a leader and I got a batch to process");
|
||||
leader.starts_batch(batch.clone().into());
|
||||
}
|
||||
}
|
||||
Ok(batch)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue