From 86c5ab16407d462ea102df618582de45954005e3 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Thu, 3 Jul 2025 12:25:28 +0200 Subject: [PATCH 1/2] chore(tesseract): Refactoring of time shift without time dimension logic --- .../logical_plan/multistage/leaf_measure.rs | 12 ++-- .../src/physical_plan_builder/builder.rs | 3 +- .../planners/multi_stage/applied_state.rs | 59 +++++++++++++++++++ .../multi_stage/member_query_planner.rs | 2 +- .../sql_evaluator/sql_nodes/factory.rs | 7 ++- .../sql_evaluator/sql_nodes/time_shift.rs | 18 +++--- 6 files changed, 79 insertions(+), 22 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs index 394c79125451c..c8f010fd132e8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs @@ -1,13 +1,14 @@ use crate::logical_plan::*; use crate::planner::planners::multi_stage::TimeShiftState; -use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::{DimensionTimeShift, MemberSymbol}; +use std::collections::HashMap; use std::rc::Rc; pub struct MultiStageLeafMeasure { pub measure: Rc, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: TimeShiftState, + pub time_shifts: HashMap, pub query: Rc, } @@ -22,13 +23,10 @@ impl PrettyPrint for MultiStageLeafMeasure { if self.render_measure_for_ungrouped { result.println("render_measure_for_ungrouped: true", &state); } - if !self.time_shifts.dimensions_shifts.is_empty() { + if !self.time_shifts.is_empty() { result.println("time_shifts:", &state); let details_state = state.new_level(); - if let Some(common) = &self.time_shifts.common_time_shift { - result.println(&format!("- common: {}", common.to_sql()), &details_state); - } - for (_, time_shift) in self.time_shifts.dimensions_shifts.iter() { + for (_, time_shift) in self.time_shifts.iter() { result.println( &format!( "- {}: {}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 2c66e0977b4a6..6329245593f12 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -5,6 +5,7 @@ use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_properties::OrderByItem; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; +use crate::planner::sql_evaluator::DimensionTimeShift; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::ReferencesBuilder; use crate::planner::sql_templates::PlanSqlTemplates; @@ -24,7 +25,7 @@ struct PhysicalPlanBuilderContext { pub alias_prefix: Option, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: TimeShiftState, + pub time_shifts: HashMap, pub original_sql_pre_aggregations: HashMap, } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index 150cbf4a85265..c3e92e54b1a35 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -104,6 +104,57 @@ impl MultiStageAppliedState { &self.time_shifts } + pub fn resolved_time_shifts(&self) -> HashMap { + let mut resolved_time_shifts = self.time_shifts.dimensions_shifts.clone(); + if let Some(common) = &self.time_shifts.common_time_shift { + for member in self.all_time_members() { + if let Some(exists) = resolved_time_shifts.get_mut(&member.full_name()) { + exists.interval += common; + } else { + let time_shift = DimensionTimeShift { + interval: common.clone(), + dimension: member.clone(), + }; + resolved_time_shifts.insert(member.full_name(), time_shift); + } + } + } + resolved_time_shifts + } + + fn all_time_members(&self) -> Vec> { + let mut filter_symbols = self.all_dimensions_symbols(); + for filter_item in self + .time_dimensions_filters + .iter() + .chain(self.dimensions_filters.iter()) + .chain(self.segments.iter()) + { + filter_item.find_all_member_evaluators(&mut filter_symbols); + } + + let time_symbols = filter_symbols + .into_iter() + .filter_map(|m| { + let symbol = if let Ok(time_dim) = m.as_time_dimension() { + time_dim.base_symbol().clone().resolve_reference_chain() + } else { + m.resolve_reference_chain() + }; + if let Ok(dim) = symbol.as_dimension() { + if dim.dimension_type() == "time" { + Some(symbol) + } else { + None + } + } else { + None + } + }) + .collect_vec(); + time_symbols + } + pub fn time_dimensions_filters(&self) -> &Vec { &self.time_dimensions_filters } @@ -122,6 +173,14 @@ impl MultiStageAppliedState { .collect() } + pub fn all_dimensions_symbols(&self) -> Vec> { + self.time_dimensions + .iter() + .map(|d| d.member_evaluator().clone()) + .chain(self.dimensions.iter().map(|d| d.member_evaluator().clone())) + .collect() + } + pub fn dimensions_filters(&self) -> &Vec { &self.dimensions_filters } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index a481a23c26dba..aed1d2d7b603f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -301,7 +301,7 @@ impl MultiStageMemberQueryPlanner { measure: member_node.clone(), query, render_measure_as_state: self.description.member().has_aggregates_on_top(), - time_shifts: self.description.state().time_shifts().clone(), + time_shifts: self.description.state().resolved_time_shifts(), render_measure_for_ungrouped: self.description.member().is_ungrupped(), }; let result = LogicalMultiStageMember { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index e14749ce2007d..24fa04fccb0bb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -7,12 +7,13 @@ use super::{ }; use crate::plan::schema::QualifiedColumnName; use crate::planner::planners::multi_stage::TimeShiftState; +use crate::planner::sql_evaluator::DimensionTimeShift; use std::collections::{HashMap, HashSet}; use std::rc::Rc; #[derive(Clone)] pub struct SqlNodesFactory { - time_shifts: TimeShiftState, + time_shifts: HashMap, ungrouped: bool, ungrouped_measure: bool, count_approx_as_state: bool, @@ -33,7 +34,7 @@ pub struct SqlNodesFactory { impl SqlNodesFactory { pub fn new() -> Self { Self { - time_shifts: TimeShiftState::default(), + time_shifts: HashMap::new(), ungrouped: false, ungrouped_measure: false, count_approx_as_state: false, @@ -52,7 +53,7 @@ impl SqlNodesFactory { } } - pub fn set_time_shifts(&mut self, time_shifts: TimeShiftState) { + pub fn set_time_shifts(&mut self, time_shifts: HashMap) { self.time_shifts = time_shifts; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs index d658fad328906..1e26ee996c98a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs @@ -1,21 +1,23 @@ use super::SqlNode; use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::DimensionTimeShift; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::SqlInterval; use cubenativeutils::CubeError; use std::any::Any; +use std::collections::HashMap; use std::rc::Rc; pub struct TimeShiftSqlNode { - shifts: TimeShiftState, + shifts: HashMap, input: Rc, } impl TimeShiftSqlNode { - pub fn new(shifts: TimeShiftState, input: Rc) -> Rc { + pub fn new(shifts: HashMap, input: Rc) -> Rc { Rc::new(Self { shifts, input }) } @@ -43,16 +45,12 @@ impl SqlNode for TimeShiftSqlNode { let res = match node.as_ref() { MemberSymbol::Dimension(ev) => { if !ev.is_reference() && ev.dimension_type() == "time" { - let mut interval = self.shifts.common_time_shift.clone().unwrap_or_default(); - if let Some(shift) = self.shifts.dimensions_shifts.get(&ev.full_name()) { - interval += &shift.interval; - } - if interval == SqlInterval::default() { - input - } else { - let shift = interval.to_sql(); + if let Some(shift) = self.shifts.get(&ev.full_name()) { + let shift = shift.interval.to_sql(); let res = templates.add_timestamp_interval(input, shift)?; format!("({})", res) + } else { + input } } else { input From 4b0fb66cbce47e3b4cbec238b7adb2f78d0b7589 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Thu, 3 Jul 2025 13:49:44 +0200 Subject: [PATCH 2/2] done --- .../src/logical_plan/multistage/common.rs | 3 - .../logical_plan/multistage/leaf_measure.rs | 7 +- .../src/physical_plan_builder/builder.rs | 3 +- .../planners/multi_stage/applied_state.rs | 69 +++++++------------ .../multi_stage/member_query_planner.rs | 2 +- .../sql_evaluator/sql_nodes/factory.rs | 7 +- .../sql_evaluator/sql_nodes/time_shift.rs | 9 +-- 7 files changed, 36 insertions(+), 64 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs index 7a3b86859b438..5ca1ddd776f6a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs @@ -53,9 +53,6 @@ impl PrettyPrint for MultiStageAppliedState { } result.println("time_shifts:", &state); - if let Some(common) = &self.time_shifts().common_time_shift { - result.println(&format!("- common: {}", common.to_sql()), &details_state); - } for (_, time_shift) in self.time_shifts().dimensions_shifts.iter() { result.println( &format!( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs index c8f010fd132e8..214ab412fb8d8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs @@ -1,14 +1,13 @@ use crate::logical_plan::*; use crate::planner::planners::multi_stage::TimeShiftState; -use crate::planner::sql_evaluator::{DimensionTimeShift, MemberSymbol}; -use std::collections::HashMap; +use crate::planner::sql_evaluator::MemberSymbol; use std::rc::Rc; pub struct MultiStageLeafMeasure { pub measure: Rc, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: HashMap, + pub time_shifts: TimeShiftState, pub query: Rc, } @@ -26,7 +25,7 @@ impl PrettyPrint for MultiStageLeafMeasure { if !self.time_shifts.is_empty() { result.println("time_shifts:", &state); let details_state = state.new_level(); - for (_, time_shift) in self.time_shifts.iter() { + for (_, time_shift) in self.time_shifts.dimensions_shifts.iter() { result.println( &format!( "- {}: {}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 6329245593f12..2c66e0977b4a6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -5,7 +5,6 @@ use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_properties::OrderByItem; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; -use crate::planner::sql_evaluator::DimensionTimeShift; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::ReferencesBuilder; use crate::planner::sql_templates::PlanSqlTemplates; @@ -25,7 +24,7 @@ struct PhysicalPlanBuilderContext { pub alias_prefix: Option, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: HashMap, + pub time_shifts: TimeShiftState, pub original_sql_pre_aggregations: HashMap, } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index c3e92e54b1a35..8ccbec0c29b68 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -1,7 +1,7 @@ use crate::plan::{FilterGroup, FilterItem}; use crate::planner::filter::FilterOperator; use crate::planner::sql_evaluator::{DimensionTimeShift, MeasureTimeShifts, MemberSymbol}; -use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension, SqlInterval}; +use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension}; use itertools::Itertools; use std::cmp::PartialEq; use std::collections::HashMap; @@ -11,12 +11,11 @@ use std::rc::Rc; #[derive(Clone, Default, Debug)] pub struct TimeShiftState { pub dimensions_shifts: HashMap, - pub common_time_shift: Option, } impl TimeShiftState { pub fn is_empty(&self) -> bool { - self.dimensions_shifts.is_empty() && self.common_time_shift.is_none() + self.dimensions_shifts.is_empty() } } @@ -74,28 +73,28 @@ impl MultiStageAppliedState { } pub fn add_time_shifts(&mut self, time_shifts: MeasureTimeShifts) { - match time_shifts { - MeasureTimeShifts::Dimensions(dimensions) => { - for ts in dimensions.into_iter() { - if let Some(exists) = self - .time_shifts - .dimensions_shifts - .get_mut(&ts.dimension.full_name()) - { - exists.interval += ts.interval; - } else { - self.time_shifts - .dimensions_shifts - .insert(ts.dimension.full_name(), ts); - } - } - } - MeasureTimeShifts::Common(interval) => { - if let Some(common) = self.time_shifts.common_time_shift.as_mut() { - *common += interval; - } else { - self.time_shifts.common_time_shift = Some(interval); - } + let resolved_shifts = match time_shifts { + MeasureTimeShifts::Dimensions(dimensions) => dimensions, + MeasureTimeShifts::Common(interval) => self + .all_time_members() + .into_iter() + .map(|m| DimensionTimeShift { + interval: interval.clone(), + dimension: m, + }) + .collect_vec(), + }; + for ts in resolved_shifts.into_iter() { + if let Some(exists) = self + .time_shifts + .dimensions_shifts + .get_mut(&ts.dimension.full_name()) + { + exists.interval += ts.interval; + } else { + self.time_shifts + .dimensions_shifts + .insert(ts.dimension.full_name(), ts); } } } @@ -104,24 +103,6 @@ impl MultiStageAppliedState { &self.time_shifts } - pub fn resolved_time_shifts(&self) -> HashMap { - let mut resolved_time_shifts = self.time_shifts.dimensions_shifts.clone(); - if let Some(common) = &self.time_shifts.common_time_shift { - for member in self.all_time_members() { - if let Some(exists) = resolved_time_shifts.get_mut(&member.full_name()) { - exists.interval += common; - } else { - let time_shift = DimensionTimeShift { - interval: common.clone(), - dimension: member.clone(), - }; - resolved_time_shifts.insert(member.full_name(), time_shift); - } - } - } - resolved_time_shifts - } - fn all_time_members(&self) -> Vec> { let mut filter_symbols = self.all_dimensions_symbols(); for filter_item in self @@ -151,6 +132,7 @@ impl MultiStageAppliedState { None } }) + .unique_by(|s| s.full_name()) .collect_vec(); time_symbols } @@ -402,7 +384,6 @@ impl PartialEq for MultiStageAppliedState { && self.time_dimensions_filters == other.time_dimensions_filters && self.dimensions_filters == other.dimensions_filters && self.measures_filters == other.measures_filters - && self.time_shifts.common_time_shift == other.time_shifts.common_time_shift && self.time_shifts.dimensions_shifts == other.time_shifts.dimensions_shifts } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index aed1d2d7b603f..a481a23c26dba 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -301,7 +301,7 @@ impl MultiStageMemberQueryPlanner { measure: member_node.clone(), query, render_measure_as_state: self.description.member().has_aggregates_on_top(), - time_shifts: self.description.state().resolved_time_shifts(), + time_shifts: self.description.state().time_shifts().clone(), render_measure_for_ungrouped: self.description.member().is_ungrupped(), }; let result = LogicalMultiStageMember { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index 24fa04fccb0bb..e14749ce2007d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -7,13 +7,12 @@ use super::{ }; use crate::plan::schema::QualifiedColumnName; use crate::planner::planners::multi_stage::TimeShiftState; -use crate::planner::sql_evaluator::DimensionTimeShift; use std::collections::{HashMap, HashSet}; use std::rc::Rc; #[derive(Clone)] pub struct SqlNodesFactory { - time_shifts: HashMap, + time_shifts: TimeShiftState, ungrouped: bool, ungrouped_measure: bool, count_approx_as_state: bool, @@ -34,7 +33,7 @@ pub struct SqlNodesFactory { impl SqlNodesFactory { pub fn new() -> Self { Self { - time_shifts: HashMap::new(), + time_shifts: TimeShiftState::default(), ungrouped: false, ungrouped_measure: false, count_approx_as_state: false, @@ -53,7 +52,7 @@ impl SqlNodesFactory { } } - pub fn set_time_shifts(&mut self, time_shifts: HashMap) { + pub fn set_time_shifts(&mut self, time_shifts: TimeShiftState) { self.time_shifts = time_shifts; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs index 1e26ee996c98a..2f30043a0df38 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs @@ -1,23 +1,20 @@ use super::SqlNode; use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::DimensionTimeShift; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_templates::PlanSqlTemplates; -use crate::planner::SqlInterval; use cubenativeutils::CubeError; use std::any::Any; -use std::collections::HashMap; use std::rc::Rc; pub struct TimeShiftSqlNode { - shifts: HashMap, + shifts: TimeShiftState, input: Rc, } impl TimeShiftSqlNode { - pub fn new(shifts: HashMap, input: Rc) -> Rc { + pub fn new(shifts: TimeShiftState, input: Rc) -> Rc { Rc::new(Self { shifts, input }) } @@ -45,7 +42,7 @@ impl SqlNode for TimeShiftSqlNode { let res = match node.as_ref() { MemberSymbol::Dimension(ev) => { if !ev.is_reference() && ev.dimension_type() == "time" { - if let Some(shift) = self.shifts.get(&ev.full_name()) { + if let Some(shift) = self.shifts.dimensions_shifts.get(&ev.full_name()) { let shift = shift.interval.to_sql(); let res = templates.add_timestamp_interval(input, shift)?; format!("({})", res)