feat(cubesqlplanner): Base filters support

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -639,6 +639,7 @@ export class BaseQuery {
       timezone: this.options.timezone,
       joinRoot: this.join.root,
       cubeEvaluator: this.cubeEvaluator,
+      filters: this.options.filters,
       baseTools: this,
 
     };
@@ -3089,6 +3090,15 @@ export class BaseQuery {
         true: 'TRUE',
         false: 'FALSE',
       },
+      filters: {
+        equals: '{{ column }} = {{ value }}{{ is_null_check }}',
+        not_equals: '{{ column }} <> {{ value }}{{ is_null_check }}',
+        or_is_null_check: ' OR {{ column }} IS NULL',
+        set_where: '{{ column }} IS NOT NULL',
+        not_set_where: '{{ column }} IS NULL',
+        in: '{{ column }} IN ({{ values_concat }}){{ is_null_check }}',
+        not_in: '{{ column }} NOT IN ({{ values_concat }}){{ is_null_check }}'
+      },
       quotes: {
         identifiers: '"',
         escape: '""'

packages/cubejs-schema-compiler/test/unit/base-query.test.ts

@@ -198,6 +198,164 @@ describe('SQL Generation', () => {
           '      card_tbl AS "cards"  GROUP BY 1 ORDER BY 2 DESC';
       expect(queryAndParams[0]).toEqual(expected);
     });
+    it('Simple query - simple filter', async () => {
+      await compilers.compiler.compile();
+
+      const query = new PostgresQuery(compilers, {
+        dimensions: [
+          'cards.type'
+        ],
+        measures: [
+          'cards.count'
+        ],
+        filters: [
+          {
+            or: [
+              {
+                member: 'cards.type',
+                operator: 'equals',
+                values: ['type_value']
+              },
+              {
+                member: 'cards.type',
+                operator: 'notEquals',
+                values: ['not_type_value']
+              },
+
+            ]
+
+          },
+          {
+            member: 'cards.type',
+            operator: 'equals',
+            values: ['type_value']
+          }],
+      });
+
+      const queryAndParams = query.buildSqlAndParams();
+      const expected = 'SELECT\n' +
+          '      "cards".type "cards__type", count("cards".id) "cards__count"\n' +
+          '    FROM\n' +
+          '      card_tbl AS "cards"  WHERE (("cards".type = $1) OR ("cards".type <> $2 OR "cards".type IS NULL)) AND ("cards".type = $3) GROUP BY 1 ORDER BY 2 DESC';
+      expect(queryAndParams[0]).toEqual(expected);
+      const expectedParams = ['type_value', 'not_type_value', 'type_value'];
+      expect(queryAndParams[1]).toEqual(expectedParams);
+    });
+    it('Simple query - null and many equals filter', async () => {
+      await compilers.compiler.compile();
+
+      const query = new PostgresQuery(compilers, {
+        dimensions: [
+          'cards.type'
+        ],
+        measures: [
+          'cards.count'
+        ],
+        filters: [
+          {
+            or: [
+              {
+                member: 'cards.type',
+                operator: 'equals',
+                values: [null]
+              },
+              {
+                member: 'cards.type',
+                operator: 'notEquals',
+                values: [null]
+              },
+
+            ]
+
+          },
+          {
+            or: [
+              {
+                member: 'cards.type',
+                operator: 'equals',
+                values: ['t1', 't2']
+              },
+              {
+                member: 'cards.type',
+                operator: 'notEquals',
+                values: ['t1', 't2']
+              },
+
+            ]
+
+          },
+          {
+            or: [
+              {
+                member: 'cards.type',
+                operator: 'equals',
+                values: ['t1', null, 't2']
+              },
+              {
+                member: 'cards.type',
+                operator: 'notEquals',
+                values: ['t1', null, 't2']
+              },
+
+            ]
+
+          },
+        ],
+      });
+
+      const queryAndParams = query.buildSqlAndParams();
+      const expected = 'SELECT\n' +
+          '      "cards".type "cards__type", count("cards".id) "cards__count"\n' +
+          '    FROM\n' +
+          '      card_tbl AS "cards"  WHERE (("cards".type IS NULL) OR ("cards".type IS NOT NULL)) AND (("cards".type IN ($1, $2)) OR ("cards".type NOT IN ($3, $4) OR "cards".type IS NULL)) AND (("cards".type IN ($5, $6) OR "cards".type IS NULL) OR ("cards".type NOT IN ($7, $8))) GROUP BY 1 ORDER BY 2 DESC';
+      expect(queryAndParams[0]).toEqual(expected);
+      // let expectedParams = [ 'type_value', 'not_type_value', 'type_value' ];
+      // expect(queryAndParams[1]).toEqual(expectedParams);
+    });
+
+    it('Simple query - dimension and measure filter', async () => {
+      await compilers.compiler.compile();
+
+      const query = new PostgresQuery(compilers, {
+        dimensions: [
+          'cards.type'
+        ],
+        measures: [
+          'cards.count'
+        ],
+        filters: [
+          {
+            or: [
+              {
+                member: 'cards.type',
+                operator: 'equals',
+                values: ['type_value']
+              },
+              {
+                member: 'cards.type',
+                operator: 'notEquals',
+                values: ['not_type_value']
+              },
+
+            ]
+
+          },
+          {
+            member: 'cards.count',
+            operator: 'equals',
+            values: ['3']
+          }],
+      });
+
+      const queryAndParams = query.buildSqlAndParams();
+      const expected = 'SELECT\n' +
+          '      "cards".type "cards__type", count("cards".id) "cards__count"\n' +
+          '    FROM\n' +
+          '      card_tbl AS "cards"  WHERE (("cards".type = $1) OR ("cards".type <> $2 OR "cards".type IS NULL)) GROUP BY 1 HAVING (count("cards".id) = $3) ORDER BY 2 DESC';
+      expect(queryAndParams[0]).toEqual(expected);
+      const expectedParams = ['type_value', 'not_type_value', '3'];
+      expect(queryAndParams[1]).toEqual(expectedParams);
+    });
   });
 
   describe('Common - JS', () => {

rust/cubesqlplanner/cubesqlplanner/Cargo.toml

@@ -15,6 +15,7 @@ async-trait = "0.1.36"
 serde = "1.0.115"
 serde_json = "1.0.56"
 cubenativeutils = { path = "../../cubenativeutils/" }
+minijinja = { version = "1", features = ["json", "loader"] }
 convert_case = "0.6"
 lazy_static = "1.4.0"
 regex = "1.3.9"

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -17,13 +17,23 @@ pub struct TimeDimension {
     pub date_range: Vec<String>,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct FilterItem {
+    pub or: Option<Vec<FilterItem>>,
+    pub and: Option<Vec<FilterItem>>,
+    pub member: Option<String>,
+    pub operator: Option<String>,
+    pub values: Option<Vec<Option<String>>>,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct BaseQueryOptionsStatic {
     pub measures: Option<Vec<String>>,
     pub dimensions: Option<Vec<String>>,
     #[serde(rename = "timeDimensions")]
     pub time_dimensions: Option<Vec<TimeDimension>>,
     pub timezone: Option<String>,
+    pub filters: Option<Vec<FilterItem>>,
     #[serde(rename = "joinRoot")]
     pub join_root: Option<String>, //TODO temporaty. join graph should be rewrited in rust or taked
                                    //from Js CubeCompiller

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -1,6 +1,7 @@
 use super::cube_definition::{CubeDefinition, NativeCubeDefinition};
 use super::dimension_definition::{DimensionDefinition, NativeDimensionDefinition};
 use super::measure_definition::{MeasureDefinition, NativeMeasureDefinition};
+use super::sql_templates_render::{NativeSqlTemplatesRender, SqlTemplatesRender};
 use cubenativeutils::wrappers::serializer::{
     NativeDeserialize, NativeDeserializer, NativeSerialize,
 };
@@ -19,4 +20,5 @@ pub trait BaseTools {
         granularity: String,
         dimension: String,
     ) -> Result<String, CubeError>;
+    fn sql_templates(&self) -> Result<Rc<dyn SqlTemplatesRender>, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs

@@ -6,3 +6,4 @@ pub mod evaluator;
 pub mod measure_definition;
 pub mod member_definition;
 pub mod memeber_sql;
+pub mod sql_templates_render;

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/sql_templates_render.rs

@@ -0,0 +1,73 @@
+use cubenativeutils::wrappers::inner_types::InnerTypes;
+use cubenativeutils::wrappers::serializer::{
+    NativeDeserialize, NativeDeserializer, NativeSerialize,
+};
+use cubenativeutils::wrappers::NativeObjectHandle;
+use cubenativeutils::CubeError;
+use minijinja::{context, value::Value, Environment};
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+pub trait SqlTemplatesRender {
+    fn contains_template(&self, template_name: &str) -> bool;
+    fn render_template(&self, name: &str, ctx: Value) -> Result<String, CubeError>;
+}
+
+pub struct NativeSqlTemplatesRender<IT: InnerTypes> {
+    templates: HashMap<String, String>,
+    jinja: Environment<'static>,
+    phantom: PhantomData<IT>,
+}
+
+impl<IT: InnerTypes> NativeSqlTemplatesRender<IT> {
+    pub fn try_new(templates: HashMap<String, String>) -> Result<Self, CubeError> {
+        let mut jinja = Environment::new();
+        for (name, template) in templates.iter() {
+            jinja
+                .add_template_owned(name.to_string(), template.to_string())
+                .map_err(|e| {
+                    CubeError::internal(format!(
+                        "Error parsing template {} '{}': {}",
+                        name, template, e
+                    ))
+                })?;
+        }
+
+        Ok(Self {
+            templates,
+            jinja,
+            phantom: PhantomData::default(),
+        })
+    }
+}
+
+impl<IT: InnerTypes> SqlTemplatesRender for NativeSqlTemplatesRender<IT> {
+    fn contains_template(&self, template_name: &str) -> bool {
+        self.templates.contains_key(template_name)
+    }
+
+    fn render_template(&self, name: &str, ctx: Value) -> Result<String, CubeError> {
+        Ok(self
+            .jinja
+            .get_template(name)
+            .map_err(|e| CubeError::internal(format!("Error getting {} template: {}", name, e)))?
+            .render(ctx)
+            .map_err(|e| {
+                CubeError::internal(format!("Error rendering {} template: {}", name, e))
+            })?)
+    }
+}
+
+impl<IT: InnerTypes> NativeDeserialize<IT> for NativeSqlTemplatesRender<IT> {
+    fn from_native(native_object: NativeObjectHandle<IT>) -> Result<Self, CubeError> {
+        let raw_data = HashMap::<String, HashMap<String, String>>::from_native(native_object)?;
+        let mut templates_map = HashMap::new();
+        for (template_type, templates) in raw_data {
+            for (template_name, template) in templates {
+                templates_map.insert(format!("{}/{}", template_type, template_name), template);
+            }
+        }
+        NativeSqlTemplatesRender::try_new(templates_map)
+    }
+}