Spaces:
Build error
Build error
File size: 5,163 Bytes
84d2a97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
use std::collections::VecDeque;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use parking_lot::Mutex;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::holders::segment_holder::SegmentId;
pub mod config_mismatch_optimizer;
pub mod indexing_optimizer;
pub mod merge_optimizer;
pub mod segment_optimizer;
pub mod vacuum_optimizer;
/// Number of last trackers to keep in tracker log
///
/// Will never remove older trackers for failed or still ongoing optimizations.
const KEEP_LAST_TRACKERS: usize = 16;
/// A log of optimizer trackers holding their status
#[derive(Default, Clone, Debug)]
pub struct TrackerLog {
descriptions: VecDeque<Tracker>,
}
impl TrackerLog {
/// Register a new optimizer tracker
pub fn register(&mut self, description: Tracker) {
self.descriptions.push_back(description);
self.truncate();
}
/// Truncate and forget old trackers for successful/cancelled optimizations
///
/// Will never remove older trackers with failed or still ongoing optimizations.
///
/// Always keeps the last `KEEP_TRACKERS` trackers.
fn truncate(&mut self) {
let truncate_range = self.descriptions.len().saturating_sub(KEEP_LAST_TRACKERS);
// Find items to truncate, start removing from the back
let truncate = self
.descriptions
.iter()
.enumerate()
.take(truncate_range)
.filter(|(_, tracker)| match tracker.state.lock().status {
TrackerStatus::Optimizing | TrackerStatus::Error(_) => false,
TrackerStatus::Done | TrackerStatus::Cancelled(_) => true,
})
.map(|(index, _)| index)
.collect::<Vec<_>>();
truncate.into_iter().rev().for_each(|index| {
self.descriptions.remove(index);
});
}
/// Convert log into list of objects usable in telemetry
pub fn to_telemetry(&self) -> Vec<TrackerTelemetry> {
self.descriptions
.iter()
// Show latest items first
.rev()
.map(Tracker::to_telemetry)
.collect()
}
}
/// Tracks the state of an optimizer
#[derive(Clone, Debug)]
pub struct Tracker {
/// Name of the optimizer
pub name: String,
/// Segment IDs being optimized
pub segment_ids: Vec<SegmentId>,
/// Start time of the optimizer
pub start_at: DateTime<Utc>,
/// Latest state of the optimizer
pub state: Arc<Mutex<TrackerState>>,
}
impl Tracker {
/// Start a new optimizer tracker
pub fn start(name: impl Into<String>, segment_ids: Vec<SegmentId>) -> Self {
Self {
name: name.into(),
segment_ids,
state: Default::default(),
start_at: Utc::now(),
}
}
/// Get handle to this tracker, allows updating state
pub fn handle(&self) -> TrackerHandle {
self.state.clone().into()
}
/// Convert into object used in telemetry
pub fn to_telemetry(&self) -> TrackerTelemetry {
let state = self.state.lock();
TrackerTelemetry {
name: self.name.clone(),
segment_ids: self.segment_ids.clone(),
status: state.status.clone(),
start_at: self.start_at,
end_at: state.end_at,
}
}
}
/// Tracker object used in telemetry
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
pub struct TrackerTelemetry {
/// Name of the optimizer
pub name: String,
/// Segment IDs being optimized
pub segment_ids: Vec<SegmentId>,
/// Latest status of the optimizer
pub status: TrackerStatus,
/// Start time of the optimizer
pub start_at: DateTime<Utc>,
/// End time of the optimizer
pub end_at: Option<DateTime<Utc>>,
}
/// Handle to an optimizer tracker, allows updating its state
#[derive(Clone)]
pub struct TrackerHandle {
handle: Arc<Mutex<TrackerState>>,
}
impl TrackerHandle {
pub fn update(&self, status: TrackerStatus) {
self.handle.lock().update(status);
}
}
impl From<Arc<Mutex<TrackerState>>> for TrackerHandle {
fn from(state: Arc<Mutex<TrackerState>>) -> Self {
Self { handle: state }
}
}
/// Mutable state of an optimizer tracker
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct TrackerState {
pub status: TrackerStatus,
pub end_at: Option<DateTime<Utc>>,
}
impl TrackerState {
/// Update the tracker state to the given `status`
pub fn update(&mut self, status: TrackerStatus) {
match status {
TrackerStatus::Done | TrackerStatus::Cancelled(_) | TrackerStatus::Error(_) => {
self.end_at.replace(Utc::now());
}
TrackerStatus::Optimizing => {
self.end_at.take();
}
}
self.status = status;
}
}
/// Represents the current state of the optimizer being tracked
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, Default, Eq, PartialEq, Hash)]
#[serde(rename_all = "lowercase")]
pub enum TrackerStatus {
#[default]
Optimizing,
Done,
Cancelled(String),
Error(String),
}
|