From 8652ad719fa049244488a7563b325da2eaad8530 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Thu, 4 Jul 2024 19:47:39 +1000 Subject: [PATCH 01/33] IGNITE-22557 Tests added --- .../calcite/SqlPlanHistoryCalciteTest.java | 55 +++++++++++++++++++ .../processors/query/SqlPlanHistoryTest.java | 44 +++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java new file mode 100644 index 0000000000000..0c584361e5c95 --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java @@ -0,0 +1,55 @@ +package org.apache.ignite.internal.processors.query.calcite; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.IndexQuery; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +public class SqlPlanHistoryCalciteTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setSqlConfiguration( + new SqlConfiguration().setQueryEnginesConfiguration(new CalciteQueryEngineConfiguration())); + } + + @Test + public void test() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = ignite.getOrCreateCache( + new CacheConfiguration("test") + .setQueryEntities(Collections.singleton( + new QueryEntity() + .setTableName("test") + .setKeyType(Integer.class.getName()) + .setValueType(Integer.class.getName())))); + + for (int i = 0; i < 5; i++) + cache.put(i, i); + + Iterator> iter1 = cache.query( + new SqlFieldsQuery("SELECT * FROM test").setLocal(false)).iterator(); + +// Iterator> iter2 = cache.query( +// new ScanQuery<>()).iterator(); + +// Iterator> iter3 = cache.query( +// new IndexQuery<>(Integer.class)).iterator(); + + assertTrue(iter1.hasNext()); +// assertTrue(iter2.hasNext()); +// assertTrue(iter3.hasNext()); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java new file mode 100644 index 0000000000000..05506658aeddc --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java @@ -0,0 +1,44 @@ +package org.apache.ignite.internal.processors.query; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +public class SqlPlanHistoryTest extends GridCommonAbstractTest { + + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setSqlConfiguration( + new SqlConfiguration().setQueryEnginesConfiguration(new IndexingQueryEngineConfiguration())); + } + + @Test + public void test() throws Exception { + IgniteEx ignite = startGrids(2); + + IgniteCache cache = ignite.getOrCreateCache( + new CacheConfiguration("test") + .setQueryEntities(Collections.singleton( + new QueryEntity() + .setTableName("test") + .setKeyType(Integer.class.getName()) + .setValueType(Integer.class.getName())))); + + for (int i = 0; i < 5; i++) + cache.put(i, i); + + Iterator> iter1 = cache.query( + new SqlFieldsQuery("SELECT * FROM test").setLocal(false)).iterator(); + + assertTrue(iter1.hasNext()); + } +} From 7485803e39d130df93990591a7056e03ee0e140b Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 12:43:32 +1000 Subject: [PATCH 02/33] IGNITE-22557 SQL Plan History view introduced --- modules/calcite/pom.xml | 6 + .../processors/query/calcite/RootQuery.java | 5 + .../calcite/exec/ExecutionServiceImpl.java | 12 + .../src/test/config/plan-history-conf.xml | 33 + ...lPlanHistoryCalciteFromClientSelfTest.java | 39 + .../SqlPlanHistoryCalciteSelfTest.java | 29 + .../calcite/SqlPlanHistoryCalciteTest.java | 55 -- .../calcite/SqlPlanHistoryConfigTest.java | 41 ++ .../SqlPlanHistoryH2FromClientSelfTest.java | 39 + .../calcite/SqlPlanHistoryH2SelfTest.java | 678 ++++++++++++++++++ .../jdbc/thin/JdbcThinMetadataSelfTest.java | 11 +- ...SystemViewRowAttributeWalkerGenerator.java | 2 + .../configuration/SqlConfiguration.java | 30 + .../walker/SqlPlanHistoryViewWalker.java | 55 ++ .../query/running/RunningQueryManager.java | 35 + .../processors/query/running/SqlPlan.java | 87 +++ .../query/running/SqlPlanHistoryTracker.java | 74 ++ .../processors/query/running/SqlPlanKey.java | 95 +++ .../query/running/SqlPlanValue.java | 46 ++ .../systemview/view/SqlPlanHistoryView.java | 83 +++ modules/indexing/pom.xml | 6 + .../processors/query/h2/H2QueryInfo.java | 5 + .../processors/query/h2/IgniteH2Indexing.java | 23 + .../processors/query/h2/dml/DmlArguments.java | 7 +- .../processors/query/h2/dml/FastUpdate.java | 15 + .../processors/query/h2/dml/UpdatePlan.java | 91 ++- .../h2/twostep/GridMapQueryExecutor.java | 12 + .../h2/twostep/GridReduceQueryExecutor.java | 12 + .../src/test/config/plan-history-conf.xml | 33 + .../cache/metric/SqlViewExporterSpiTest.java | 3 +- .../query/SqlPlanHistoryConfigTest.java | 41 ++ .../processors/query/SqlPlanHistoryTest.java | 44 -- 32 files changed, 1640 insertions(+), 107 deletions(-) create mode 100644 modules/calcite/src/test/config/plan-history-conf.xml create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java delete mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java create mode 100644 modules/indexing/src/test/config/plan-history-conf.xml create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java diff --git a/modules/calcite/pom.xml b/modules/calcite/pom.xml index d80b33b8258d3..d4315a4b26955 100644 --- a/modules/calcite/pom.xml +++ b/modules/calcite/pom.xml @@ -64,6 +64,12 @@ test + + ${project.groupId} + ignite-spring + test + + org.apache.calcite calcite-core diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java index 242e355d22c7e..951c9490a5fdb 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java @@ -456,6 +456,11 @@ public long remainingTime() { return curTimeout <= 0 ? 0 : curTimeout; } + /** */ + public long startTime() { + return startTs; + } + /** */ @Override public String toString() { return S.toString(RootQuery.class, this); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java index bca13901aafd4..654ad847033cb 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java @@ -110,6 +110,7 @@ import org.apache.ignite.internal.processors.query.calcite.util.ConvertingClosableIterator; import org.apache.ignite.internal.processors.query.calcite.util.ListFieldsQueryCursor; import org.apache.ignite.internal.processors.query.running.HeavyQueriesTracker; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.security.SecurityUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; @@ -685,6 +686,17 @@ private ListFieldsQueryCursor mapAndExecutePlan( ); } + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + plan.textPlan(), + qry.sql(), + qry.context().schemaName(), + qry.context().isLocal(), + qry.startTime(), + SqlPlanHistoryTracker.SqlEngine.CALCITE + ); + QueryProperties qryProps = qry.context().unwrap(QueryProperties.class); Function fieldConverter = (qryProps == null || qryProps.keepBinary()) ? null : diff --git a/modules/calcite/src/test/config/plan-history-conf.xml b/modules/calcite/src/test/config/plan-history-conf.xml new file mode 100644 index 0000000000000..959be5045ff53 --- /dev/null +++ b/modules/calcite/src/test/config/plan-history-conf.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java new file mode 100644 index 0000000000000..9b17a65c458f4 --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite; + +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.JUnitAssertAware; + +/** Tests for SQL plan history from client (Calcite engine). */ +public class SqlPlanHistoryCalciteFromClientSelfTest extends SqlPlanHistoryCalciteSelfTest { + /** {@inheritDoc} */ + @Override protected IgniteEx queryNode() { + IgniteEx node = grid(1); + + JUnitAssertAware.assertTrue(node.context().clientNode()); + + return node; + } + + /** {@inheritDoc} */ + @Override protected void startTestGrid() throws Exception { + startGrid(0); + startClientGrid(1); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java new file mode 100644 index 0000000000000..52940c6ca0c8f --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite; + +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; + +/** Tests for SQL plan history (Calcite engine). */ +public class SqlPlanHistoryCalciteSelfTest extends SqlPlanHistoryH2SelfTest { + /** {@inheritDoc} */ + @Override protected QueryEngineConfigurationEx configureSqlEngine() { + return new CalciteQueryEngineConfiguration(); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java deleted file mode 100644 index 0c584361e5c95..0000000000000 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.apache.ignite.internal.processors.query.calcite; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import javax.cache.Cache; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.query.IndexQuery; -import org.apache.ignite.cache.query.ScanQuery; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.SqlConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -public class SqlPlanHistoryCalciteTest extends GridCommonAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName).setSqlConfiguration( - new SqlConfiguration().setQueryEnginesConfiguration(new CalciteQueryEngineConfiguration())); - } - - @Test - public void test() throws Exception { - IgniteEx ignite = startGrid(1); - - IgniteCache cache = ignite.getOrCreateCache( - new CacheConfiguration("test") - .setQueryEntities(Collections.singleton( - new QueryEntity() - .setTableName("test") - .setKeyType(Integer.class.getName()) - .setValueType(Integer.class.getName())))); - - for (int i = 0; i < 5; i++) - cache.put(i, i); - - Iterator> iter1 = cache.query( - new SqlFieldsQuery("SELECT * FROM test").setLocal(false)).iterator(); - -// Iterator> iter2 = cache.query( -// new ScanQuery<>()).iterator(); - -// Iterator> iter3 = cache.query( -// new IndexQuery<>(Integer.class)).iterator(); - - assertTrue(iter1.hasNext()); -// assertTrue(iter2.hasNext()); -// assertTrue(iter3.hasNext()); - } -} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java new file mode 100644 index 0000000000000..10f6f347ca3fe --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite; + +import org.apache.ignite.Ignite; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** Test for Sql plan history configuration. */ +public class SqlPlanHistoryConfigTest extends GridCommonAbstractTest { + /** Sql plan history size in the XML Spring config. */ + private static final int SQL_PLAN_HISTORY_SIZE_XML_CONFIG = 10; + + /** Checks that plan history size specified in XML config is respected. */ + @Test + public void testXmlConfigSqlPlanHistorySize() throws Exception { + String cfgPath = "modules/calcite/src/test/config/plan-history-conf.xml"; + + Ignite ignite = startGridsWithSpringCtx(2, false, cfgPath); + + assertEquals(SQL_PLAN_HISTORY_SIZE_XML_CONFIG, + ignite.configuration().getSqlConfiguration().getSqlPlanHistorySize()); + + stopAllGrids(); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java new file mode 100644 index 0000000000000..a5ae70e8413ed --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite; + +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.JUnitAssertAware; + +/** Tests for SQL plan history from client (H2 engine). */ +public class SqlPlanHistoryH2FromClientSelfTest extends SqlPlanHistoryH2SelfTest { + /** {@inheritDoc} */ + @Override protected IgniteEx queryNode() { + IgniteEx node = grid(1); + + JUnitAssertAware.assertTrue(node.context().clientNode()); + + return node; + } + + /** {@inheritDoc} */ + @Override protected void startTestGrid() throws Exception { + startGrid(0); + startClientGrid(1); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java new file mode 100644 index 0000000000000..495378118441d --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java @@ -0,0 +1,678 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; +import org.apache.ignite.internal.processors.query.running.SqlPlan; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** Tests for SQL plan history (H2 engine). */ +@RunWith(Parameterized.class) +public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { + /** SQL Plan history size. */ + private static final int PLAN_HISTORY_SIZE = 10; + + /** Plan history size excess. */ + private static final int PLAN_HISTORY_EXCESS = 2; + + /** Simple query. */ + private static final String SQL = "SELECT * FROM A.String"; + + /** Failed query. */ + private static final String SQL_FAILED = "select * from A.String where A.fail()=1"; + + /** Cross-cache query. */ + private static final String SQL_CROSS_CACHE = "SELECT * FROM B.String"; + + /** Failed cross-cache query. */ + private static final String SQL_CROSS_CACHE_FAILED = "select * from A.String where A.fail()=1"; + + /** Query with reduce phase. */ + private static final String SQL_WITH_REDUCE_PHASE = "select o.name n1, p.name n2 from \"pers\".Person p, " + + "\"org\".Organization o where p.orgId=o._key and o._key=101" + + " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + + " where p.orgId=o._key and o._key=102"; + + /** Sets of DML commands of various complexity including failed commands. */ + List> dmlCmds = Arrays.asList( + Arrays.asList( + "insert into A.String (_key, _val) values(101, '101')", + "update A.String set _val='111' where _key=101", + "delete from A.String where _key=101" + ), + Arrays.asList( + "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", + "update A.String set _val = case _key " + + "when 101 then '111' " + + "when 102 then '112' " + + "when 103 then '113' " + + "end " + + "where _key in (101, 102, 103)", + "delete from A.String where _key in (101, 102, 103)" + ), + Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name " + + "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", + "update A.String set _val = 'updated' where _key in " + + "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", + "delete from A.String where _key in (select orgId from \"pers\".Person)" + ), + Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + + "where A.fail()=1", + "update A.String set _val = 'failed' where A.fail()=1", + "delete from A.String where A.fail()=1" + ) + ); + + /** Strings for checking SQL plan history after executing DML commands. */ + List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); + + /** Successful SqlFields query. */ + SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); + + /** Failed SqlFields query. */ + SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); + + /** Successful cross-cache SqlFields query. */ + SqlFieldsQuery sqlFieldsCrossCacheQry = new SqlFieldsQuery(SQL_CROSS_CACHE); + + /** Failed cross-cache SqlFields query. */ + SqlFieldsQuery sqlFieldsCrossCacheQryFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); + + /** Successful Sql query. */ + SqlQuery sqlQry = new SqlQuery<>("String", "from String"); + + /** Failed Sql query. */ + SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); + + /** Scan query. */ + ScanQuery scanQry = new ScanQuery<>(); + + /** Text query. */ + TextQuery textQry = new TextQuery<>("String", "2"); + + /** */ + private final SqlFieldsQuery sqlFieldsQryWithReducePhase = new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE) + .setDistributedJoins(true); + + /** */ + private final SqlFieldsQuery[] sqlFieldsQryWithReducePhaseFailed = F.asArray( + new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=101", "fail()")).setDistributedJoins(true), + new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=102", "fail()")).setDistributedJoins(true) + ); + + /** Client mode flag. */ + private boolean isClient; + + /** Sql engine. */ + private SqlPlanHistoryTracker.SqlEngine sqlEngine; + + /** Local query flag. */ + @Parameterized.Parameter + public boolean loc; + + /** Fully fetched query flag. */ + @Parameterized.Parameter(1) + public boolean isFullyFetched; + + /** */ + @Parameterized.Parameters(name = "loc={0}, fullyFetched={1}") + public static Collection params() { + return Arrays.asList(new Object[][] { + {true, true}, {true, false}, {false, true}, {false, false} + }); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + QueryEngineConfigurationEx engCfg = configureSqlEngine(); + + cfg.setSqlConfiguration(new SqlConfiguration() + .setSqlPlanHistorySize(PLAN_HISTORY_SIZE) + .setQueryEnginesConfiguration(engCfg) + ); + + return cfg.setCacheConfiguration( + configureCahce("A", Integer.class, String.class), + configureCahce("B", Integer.class, String.class), + configureCahce("pers", Integer.class, Person.class), + configureCahce("org", Integer.class, Organization.class) + ); + } + + /** */ + protected QueryEngineConfigurationEx configureSqlEngine() { + return new IndexingQueryEngineConfiguration(); + } + + /** + * @param name Name. + * @param idxTypes Index types. + * @return Cache configuration. + */ + @SuppressWarnings("unchecked") + private CacheConfiguration configureCahce(String name, Class... idxTypes) { + return new CacheConfiguration() + .setName(name) + .setIndexedTypes(idxTypes) + .setSqlFunctionClasses(Functions.class); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startTestGrid(); + + IgniteCache cacheA = queryNode().cache("A"); + IgniteCache cacheB = queryNode().cache("B"); + + for (int i = 0; i < 100; i++) { + cacheA.put(i, String.valueOf(i)); + cacheB.put(i, String.valueOf(i)); + } + + IgniteCache cachePers = queryNode().cache("pers"); + IgniteCache cacheOrg = queryNode().cache("org"); + + cacheOrg.put(101, new Organization("o1")); + cacheOrg.put(102, new Organization("o2")); + cachePers.put(103, new Person(101, "p1")); + cachePers.put(104, new Person(102, "p2")); + cachePers.put(105, new Person(103, "p3")); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + resetPlanHistory(); + + sqlEngine = isCalcite() ? SqlPlanHistoryTracker.SqlEngine.CALCITE : SqlPlanHistoryTracker.SqlEngine.H2; + + try { + if (grid(1).configuration().isClientMode()) + isClient = true; + } + catch (Exception ignore) { + //No-Op + } + } + + /** */ + public boolean isCalcite() { + return grid(0).context().config().getSqlConfiguration().getQueryEnginesConfiguration()[0] instanceof + CalciteQueryEngineConfiguration; + } + + /** */ + public void resetPlanHistory() { + queryNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); + } + + /** Checks successful JDBC queries. */ + @Test + public void testJdbcQuery() throws SQLException { + if (loc) + return; + + for (int i = 0; i < 2; i++) { + runJdbcQuery(SQL); + + checkSqlPlanHistory(getExpectedHistorySize()); + } + } + + /** Checks failed JDBC queries. */ + @Test + public void testJdbcQueryFailed() { + if (loc) + return; + + try { + runJdbcQuery(SQL_FAILED); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory(getExpectedHistorySize()); + } + + /** Checks successful SqlFields queries. */ + @Test + public void testSqlFieldsQuery() { + executeSuccessfulQuery(sqlFieldsQry); + } + + /** Checks failed SqlFields queries. */ + @Test + public void testSqlFieldsQueryFailed() { + executeFailedQuery(sqlFieldsQryFailed); + } + + /** Checks successful cross-cache SqlFields queries. */ + @Test + public void testSqlFieldsCrossCacheQuery() { + executeSuccessfulQuery(sqlFieldsCrossCacheQry); + } + + /** Checks failed cross-cache SqlFields queries. */ + @Test + public void testSqlFieldsCrossCacheQueryFailed() { + executeFailedQuery(sqlFieldsCrossCacheQryFailed); + } + + /** Checks successful SqlFields queries with reduce phase. */ + @Test + public void testSqlFieldsQueryWithReducePhase() { + if (loc) + return; + + runCacheQuery(sqlFieldsQryWithReducePhase, "pers"); + + checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); + } + + /** Checks failed SqlFields queries with reduce phase. */ + @Test + public void testSqlFieldsQueryWithReducePhaseFailed() { + if (loc) + return; + + for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { + try { + runCacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? i + 1 : 0); + + resetPlanHistory(); + } + } + + /** Checks successful Sql queries. */ + @Test + public void testSqlQuery() { + executeSuccessfulQuery(sqlQry); + } + + /** Checks failed Sql queries. */ + @Test + public void testSqlQueryFailed() { + executeFailedQuery(sqlQryFailed); + } + + /** Checks scan queries. */ + @Test + public void testScanQuery() { + executeQueryWithoutPlan(scanQry); + } + + /** Checks text queries. */ + @Test + public void testTextQuery() { + executeQueryWithoutPlan(textQry); + } + + /** Checks DML commands executed via JDBC. */ + @Test + public void testJdbcDml() throws Exception { + if (loc || !isFullyFetched) + return; + + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + for (List setOfCmds : dmlCmds) { + for (String cmd : setOfCmds) { + try { + stmt.execute(cmd); + } + catch (Exception ignore) { + //No-Op + } + } + + checkSqlPlanHistoryDml(3); + + resetPlanHistory(); + } + } + } + + /** Checks DML commands executed via SqlField queries. */ + @Test + public void testSqlFieldsDml() { + if (isClient && loc || !isFullyFetched) + return; + + for (List setOfCmds : dmlCmds) { + setOfCmds.forEach(this::runDmlCommand); + + checkSqlPlanHistoryDml(3); + + resetPlanHistory(); + } + } + + /** Checks that older plan entries are evicted when maximum history size is reached. */ + @Test + public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { + if (loc || (!loc && isFullyFetched)) + return; + + for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { + try { + runCacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); + } + catch (Exception ignore) { + //No-Op + } + } + + GridTestUtils.waitForCondition(() -> getSqlPlanHistoryValues().size() == PLAN_HISTORY_SIZE, 1000); + + checkSqlPlanHistory((isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : PLAN_HISTORY_SIZE); + + Set qrys = getSqlPlanHistoryValues().stream().map(SqlPlan::query).collect(Collectors.toSet()); + + for (int i = 1; i <= PLAN_HISTORY_EXCESS; i++) + assertFalse(qrys.contains(SQL + " where A.fail=" + i)); + } + + /** + * @param qry Query. + */ + public void executeSuccessfulQuery(Query qry) { + executeQuery(qry, () -> { + for (int i = 0; i < 2; i++) { + runCacheQuery(qry, "A"); + + checkSqlPlanHistory(getExpectedHistorySize()); + } + }); + } + + /** + * @param qry Query. + */ + public void executeFailedQuery(Query qry) { + executeQuery(qry, () -> { + try { + runCacheQuery(qry, "A"); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory(getExpectedHistorySize()); + }); + } + + /** + * @param qry Query. + */ + public void executeQueryWithoutPlan(Query qry) { + executeQuery(qry, () -> { + runCacheQuery(qry, "A"); + + checkSqlPlanHistory(0); + }); + } + + /** + * @param qry Query. + * @param task Task to execute. + */ + public void executeQuery(Query qry, Runnable task) { + if (isClient && loc) + return; + + qry.setLocal(loc); + + task.run(); + } + + /** + * @param qry Query. + */ + private void runJdbcQuery(String qry) throws SQLException { + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + if (!isFullyFetched) + stmt.setFetchSize(1); + + ResultSet rs = stmt.executeQuery(qry); + + assertTrue(rs.next()); + } + } + + /** + * @param qry Query. + * @param cacheName Cache name. + */ + public void runCacheQuery(Query qry, String cacheName) { + IgniteCache cache = queryNode().getOrCreateCache(cacheName); + + if (isFullyFetched) + assertFalse(cache.query(qry).getAll().isEmpty()); + else { + qry.setPageSize(1); + + assertTrue(cache.query(qry).iterator().hasNext()); + + cache.query(qry).iterator().next(); + } + } + + /** + * @param cmd DML command. + */ + public void runDmlCommand(String cmd) { + IgniteCache cache = queryNode().getOrCreateCache("A"); + + try { + cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); + } + catch (Exception ignore) { + //No-Op + } + } + + /** */ + public Collection getSqlPlanHistoryValues() { + return queryNode().context().query().runningQueryManager().planHistoryTracker() + .sqlPlanHistory().values(); + } + + /** */ + public int getExpectedHistorySize() { + return (isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : 1; + } + + /** + * @param size Number of SQL plan entries expected to be in the history. + */ + public void checkSqlPlanHistory(int size) { + Collection sqlPlans = getSqlPlanHistoryValues(); + + assertNotNull(sqlPlans); + + checkMetrics(size, sqlPlans); + } + + /** + * @param size Number of SQL plan entries expected to be in the history. + */ + public void checkSqlPlanHistoryDml(int size) { + Collection sqlPlans = getSqlPlanHistoryValues(); + + assertNotNull(sqlPlans); + + if (sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) { + Collection sqlPlans0 = new ArrayList<>(); + + for (String str : dmlCheckStrings) + sqlPlans0.addAll(sqlPlans.stream().filter(p -> p.plan().contains(str)).collect(Collectors.toList())); + + sqlPlans = sqlPlans0; + } + + checkMetrics(size, sqlPlans); + } + + /** + * @param size Number of SQL plan entries expected to be in the history. + * @param sqlPlans Sql plans recorded in the history. + */ + public void checkMetrics(int size, Collection sqlPlans) { + if (size == 1 && sqlPlans.size() == 2) { + String plan1 = new ArrayList<>(sqlPlans).get(0).plan(); + String plan2 = new ArrayList<>(sqlPlans).get(1).plan(); + + assertTrue(plan2.contains(plan1) && plan2.contains("/* scanCount")); + } + else + assertTrue(size == sqlPlans.size()); + + if (size == 0) + return; + + for (SqlPlan plan : sqlPlans) { + assertEquals(loc, plan.local()); + assertEquals(sqlEngine.toString(), plan.engine()); + + assertNotNull(plan.plan()); + assertNotNull(plan.query()); + assertNotNull(plan.schema()); + + assertTrue(plan.startTime() > 0); + } + } + + /** + * @return Ignite instance for quering. + */ + protected IgniteEx queryNode() { + IgniteEx node = grid(0); + + assertFalse(node.context().clientNode()); + + return node; + } + + /** + * Starts Ignite instance. + * + * @throws Exception In case of failure. + */ + protected void startTestGrid() throws Exception { + startGrid(0); + } + + /** */ + private static class Person { + /** */ + @QuerySqlField(index = true) + int orgId; + + /** */ + @QuerySqlField(index = true) + String name; + + /** + * @param orgId Organization ID. + * @param name Name. + */ + public Person(int orgId, String name) { + this.orgId = orgId; + this.name = name; + } + } + + /** */ + private static class Organization { + /** */ + @QuerySqlField + String name; + + /** + * @param name Organization name. + */ + public Organization(String name) { + this.name = name; + } + } + + /** */ + public static class Functions { + /** */ + @QuerySqlFunction + public static int fail() { + throw new IgniteSQLException("SQL function fail for test purpuses"); + } + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java index 672c5eebd41aa..60701ed03d6da 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java @@ -464,7 +464,8 @@ public void testGetAllView() throws Exception { "SYS.STATISTICS_GLOBAL_DATA", "SYS.STATISTICS_PARTITION_DATA", "SYS.STATISTICS_CONFIGURATION", - "SYS.PAGES_TIMESTAMP_HISTOGRAM" + "SYS.PAGES_TIMESTAMP_HISTOGRAM", + "SYS.SQL_PLANS_HISTORY" )) ); } @@ -1159,7 +1160,13 @@ public void testGetAllColumns() throws Exception { "SYS.PAGES_TIMESTAMP_HISTOGRAM.DATA_REGION_NAME.null", "SYS.PAGES_TIMESTAMP_HISTOGRAM.INTERVAL_START.null", "SYS.PAGES_TIMESTAMP_HISTOGRAM.INTERVAL_END.null", - "SYS.PAGES_TIMESTAMP_HISTOGRAM.PAGES_COUNT.null" + "SYS.PAGES_TIMESTAMP_HISTOGRAM.PAGES_COUNT.null", + "SYS.SQL_PLANS_HISTORY.ENGINE.null", + "SYS.SQL_PLANS_HISTORY.LAST_START_TIME.null", + "SYS.SQL_PLANS_HISTORY.LOCAL.null", + "SYS.SQL_PLANS_HISTORY.PLAN.null", + "SYS.SQL_PLANS_HISTORY.SCHEMA_NAME.null", + "SYS.SQL_PLANS_HISTORY.SQL.null" )); Assert.assertEquals(expectedCols, actualSysCols); diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java index de0be996790fb..dbc9806a175d7 100644 --- a/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java +++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java @@ -64,6 +64,7 @@ import org.apache.ignite.spi.systemview.view.ScanQueryView; import org.apache.ignite.spi.systemview.view.ServiceView; import org.apache.ignite.spi.systemview.view.SnapshotView; +import org.apache.ignite.spi.systemview.view.SqlPlanHistoryView; import org.apache.ignite.spi.systemview.view.SqlQueryHistoryView; import org.apache.ignite.spi.systemview.view.SqlQueryView; import org.apache.ignite.spi.systemview.view.StripedExecutorTaskView; @@ -126,6 +127,7 @@ public static void main(String[] args) throws Exception { gen.generateAndWrite(ScanQueryView.class, DFLT_SRC_DIR); gen.generateAndWrite(SqlQueryView.class, DFLT_SRC_DIR); gen.generateAndWrite(SqlQueryHistoryView.class, DFLT_SRC_DIR); + gen.generateAndWrite(SqlPlanHistoryView.class, DFLT_SRC_DIR); gen.generateAndWrite(StripedExecutorTaskView.class, DFLT_SRC_DIR); gen.generateAndWrite(PagesListView.class, DFLT_SRC_DIR); gen.generateAndWrite(CachePagesListView.class, DFLT_SRC_DIR); diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java index 4dd80e3e09ebf..f8afd45bb40bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java @@ -27,6 +27,9 @@ public class SqlConfiguration { /** Default SQL query history size. */ public static final int DFLT_SQL_QUERY_HISTORY_SIZE = 1000; + /** Default SQL plan history size. */ + public static final int DFLT_SQL_PLAN_HISTORY_SIZE = 1000; + /** Default query timeout. */ public static final long DFLT_QRY_TIMEOUT = 0; @@ -45,6 +48,9 @@ public class SqlConfiguration { /** SQL query history size. */ private int sqlQryHistSize = DFLT_SQL_QUERY_HISTORY_SIZE; + /** SQL plan history size. */ + private int sqlPlanHistSize = DFLT_SQL_PLAN_HISTORY_SIZE; + /** Enable validation of key & values against sql schema. */ private boolean validationEnabled; @@ -107,6 +113,30 @@ public SqlConfiguration setSqlQueryHistorySize(int size) { return this; } + /** + * Number of SQL plan history elements to keep in memory. If not provided, then default value {@link + * #DFLT_SQL_PLAN_HISTORY_SIZE} is used. If provided value is less or equals 0, then gathering SQL plan history + * will be switched off. + * + * @return SQL plan history size. + */ + public int getSqlPlanHistorySize() { + return sqlPlanHistSize; + } + + /** + * Sets number of SQL plan history elements kept in memory. If not explicitly set, then default value is {@link + * #DFLT_SQL_PLAN_HISTORY_SIZE}. + * + * @param size Number of SQL query history elements kept in memory. + * @return {@code this} for chaining. + */ + public SqlConfiguration setSqlPlanHistorySize(int size) { + sqlPlanHistSize = size; + + return this; + } + /** * Gets SQL schemas to be created on node startup. *

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java new file mode 100644 index 0000000000000..b9825180db2de --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.systemview.walker; + +import java.util.Date; +import org.apache.ignite.spi.systemview.view.SqlPlanHistoryView; +import org.apache.ignite.spi.systemview.view.SystemViewRowAttributeWalker; + +/** + * Generated by {@code org.apache.ignite.codegen.SystemViewRowAttributeWalkerGenerator}. + * {@link SqlPlanHistoryView} attributes walker. + * + * @see SqlPlanHistoryView + */ +public class SqlPlanHistoryViewWalker implements SystemViewRowAttributeWalker { + /** {@inheritDoc} */ + @Override public void visitAll(AttributeVisitor v) { + v.accept(0, "schemaName", String.class); + v.accept(1, "sql", String.class); + v.accept(2, "local", boolean.class); + v.accept(3, "engine", String.class); + v.accept(4, "lastStartTime", Date.class); + v.accept(5, "plan", String.class); + } + + /** {@inheritDoc} */ + @Override public void visitAll(SqlPlanHistoryView row, AttributeWithValueVisitor v) { + v.accept(0, "schemaName", String.class, row.schemaName()); + v.accept(1, "sql", String.class, row.sql()); + v.acceptBoolean(2, "local", row.local()); + v.accept(3, "engine", String.class, row.engine()); + v.accept(4, "lastStartTime", Date.class, row.lastStartTime()); + v.accept(5, "plan", String.class, row.plan()); + } + + /** {@inheritDoc} */ + @Override public int count() { + return 6; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java index 6d60205b90f62..699af8c9dd4a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java @@ -43,6 +43,7 @@ import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; +import org.apache.ignite.internal.managers.systemview.walker.SqlPlanHistoryViewWalker; import org.apache.ignite.internal.managers.systemview.walker.SqlQueryHistoryViewWalker; import org.apache.ignite.internal.managers.systemview.walker.SqlQueryViewWalker; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; @@ -66,6 +67,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.systemview.view.SqlPlanHistoryView; import org.apache.ignite.spi.systemview.view.SqlQueryHistoryView; import org.apache.ignite.spi.systemview.view.SqlQueryView; import org.jetbrains.annotations.Nullable; @@ -96,6 +98,12 @@ public class RunningQueryManager { /** */ public static final String SQL_QRY_HIST_VIEW_DESC = "SQL queries history."; + /** */ + public static final String SQL_PLAN_HIST_VIEW = metricName("sql", "plans", "history"); + + /** */ + public static final String SQL_PLAN_HIST_VIEW_DESC = "SQL plans history."; + /** Undefined query ID value. */ public static final long UNDEFINED_QUERY_ID = 0L; @@ -117,6 +125,12 @@ public class RunningQueryManager { /** Query history tracker. */ private volatile QueryHistoryTracker qryHistTracker; + /** Sql plan history size. */ + private final int planHistSz; + + /** Sql plan history tracker. */ + private volatile SqlPlanHistoryTracker planHistTracker; + /** Number of successfully executed queries. */ private final LongAdderMetric successQrsCnt; @@ -181,10 +195,14 @@ public RunningQueryManager(GridKernalContext ctx) { histSz = ctx.config().getSqlConfiguration().getSqlQueryHistorySize(); closure = ctx.closure(); + planHistSz = ctx.config().getSqlConfiguration().getSqlPlanHistorySize(); + qryHistTracker = new QueryHistoryTracker(histSz); heavyQrysTracker = ctx.query().moduleEnabled() ? new HeavyQueriesTracker(ctx) : null; + planHistTracker = new SqlPlanHistoryTracker(planHistSz); + ctx.systemView().registerView(SQL_QRY_VIEW, SQL_QRY_VIEW_DESC, new SqlQueryViewWalker(), runs.values(), @@ -195,6 +213,11 @@ public RunningQueryManager(GridKernalContext ctx) { qryHistTracker.queryHistory().values(), SqlQueryHistoryView::new); + ctx.systemView().registerView(SQL_PLAN_HIST_VIEW, SQL_PLAN_HIST_VIEW_DESC, + new SqlPlanHistoryViewWalker(), + planHistTracker.sqlPlanHistory().values(), + SqlPlanHistoryView::new); + MetricRegistryImpl userMetrics = ctx.metric().registry(SQL_USER_QUERIES_REG_NAME); successQrsCnt = userMetrics.longAdderMetric("success", @@ -466,6 +489,11 @@ public HeavyQueriesTracker heavyQueriesTracker() { return heavyQrysTracker; } + /** */ + public SqlPlanHistoryTracker planHistoryTracker() { + return planHistTracker; + } + /** * @param lsnr Listener. */ @@ -823,6 +851,13 @@ public void resetQueryHistoryMetrics() { qryHistTracker = new QueryHistoryTracker(histSz); } + /** + * Reset SQL plan history. + */ + public void resetPlanHistoryMetrics() { + planHistTracker = new SqlPlanHistoryTracker(planHistSz); + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(RunningQueryManager.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java new file mode 100644 index 0000000000000..87c397f4a0779 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.running; + +/** Representation of an SQL plan history entry. */ +public class SqlPlan { + /** */ + private final SqlPlanKey key; + + /** */ + private final SqlPlanValue val; + + /** + * @param plan SQL plan. + * @param qry Query. + * @param schema Schema name. + * @param loc Local query flag. + * @param startTime Start query timestamp. + */ + public SqlPlan( + String plan, + String qry, + String schema, + boolean loc, + long startTime, + SqlPlanHistoryTracker.SqlEngine engine + ) { + key = new SqlPlanKey(plan, qry, schema, loc); + + val = new SqlPlanValue(startTime, engine); + } + + /** */ + public SqlPlanKey key() { + return key; + } + + /** */ + public SqlPlanValue value() { + return val; + } + + /** */ + public String plan() { + return key.plan(); + } + + /** */ + public String query() { + return key.query(); + } + + /** */ + public String schema() { + return key.schema(); + } + + /** */ + public boolean local() { + return key.local(); + } + + /** */ + public long startTime() { + return val.startTime(); + } + + /** */ + public String engine() { + return val.engine(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java new file mode 100644 index 0000000000000..37018a55ae74d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.running; + +import java.util.Collections; +import java.util.Map; +import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; + +/** */ +public class SqlPlanHistoryTracker { + /** SQL plan history. */ + private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; + + /** SQL plan history size. */ + private final int historySize; + + /** + * @param historySize SQL plan history size. + */ + public SqlPlanHistoryTracker(int historySize) { + this.historySize = historySize; + + sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashMap<>(historySize) : null; + } + + /** + * @param plan SQL plan. + * @param qry Query. + * @param schema Schema name. + * @param loc Local query flag. + * @param startTs Start query timestamp. + * @param engine SQL engine. + */ + public void addPlan(String plan, String qry, String schema, boolean loc, long startTs, SqlEngine engine) { + if (historySize <= 0) + return; + + SqlPlan sqlPlan = new SqlPlan(plan, qry, schema, loc, startTs, engine); + + sqlPlanHistory.put(sqlPlan.key(), sqlPlan); + } + + /** */ + public Map sqlPlanHistory() { + if (historySize <= 0) + return Collections.emptyMap(); + + return Collections.unmodifiableMap(sqlPlanHistory); + } + + /** */ + public enum SqlEngine { + /** */ + CALCITE, + + /** */ + H2 + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java new file mode 100644 index 0000000000000..24c044482ae52 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.running; + +import org.apache.ignite.internal.util.typedef.F; + +/** SQL plan history entry key. */ +public class SqlPlanKey { + /** Plan. */ + private final String plan; + + /** Query. */ + private final String qry; + + /** Schema name. */ + private final String schema; + + /** Local query flag. */ + private final boolean loc; + + /** Pre-calculated hash code. */ + private final int hash; + + /** + * @param plan SQL Plan. + * @param qry Query. + * @param schema Schema name. + * @param loc Local query flag. + */ + public SqlPlanKey(String plan, String qry, String schema, boolean loc) { + assert plan != null; + assert qry != null; + assert schema != null; + + this.plan = plan; + this.qry = qry; + this.schema = schema; + this.loc = loc; + + hash = 31 * (31 * plan.hashCode() + qry.hashCode() + schema.hashCode()) + (loc ? 1 : 0); + } + + /** */ + public String plan() { + return plan; + } + + /** */ + public String query() { + return qry; + } + + /** */ + public String schema() { + return schema; + } + + /** */ + public boolean local() { + return loc; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + SqlPlanKey key = (SqlPlanKey)o; + + return F.eq(plan, key.plan) && F.eq(qry, key.qry) && F.eq(schema, key.schema) && F.eq(loc, key.loc); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return hash; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java new file mode 100644 index 0000000000000..8cb470cad9ffa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.running; + +/** SQL plan history entry key value. */ +public class SqlPlanValue { + /** Start query timestamp. */ + private final long startTime; + + /** SQL engine. */ + private final SqlPlanHistoryTracker.SqlEngine engine; + + /** + * @param startTime Start query timestamp. + * @param engine Sql engine. + */ + public SqlPlanValue(long startTime, SqlPlanHistoryTracker.SqlEngine engine) { + this.startTime = startTime; + this.engine = engine; + } + + /** */ + public long startTime() { + return startTime; + } + + /** */ + public String engine() { + return engine.name(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java new file mode 100644 index 0000000000000..f4e5681d84f38 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.systemview.view; + +import java.util.Date; +import org.apache.ignite.internal.managers.systemview.walker.Order; +import org.apache.ignite.internal.processors.query.running.SqlPlan; + +/** */ +public class SqlPlanHistoryView { + /** SQL plan. */ + private final SqlPlan plan; + + /** + * @param plan SQL plan. + */ + public SqlPlanHistoryView(SqlPlan plan) { + this.plan = plan; + } + + /** + * @return String Schema name. + */ + @Order + public String schemaName() { + return plan.schema(); + } + + /** + * @return String SQL query. + */ + @Order(1) + public String sql() { + return plan.query(); + } + + /** + * @return boolean Local query flag. + */ + @Order(2) + public boolean local() { + return plan.local(); + } + + /** + * @return String SQL engine. + */ + @Order(3) + public String engine() { + return plan.engine(); + } + + /** + * @return Date Last time the query was executed. + */ + @Order(4) + public Date lastStartTime() { + return new Date(plan.startTime()); + } + + /** + * @return String SQL plan. + */ + @Order(5) + public String plan() { + return plan.plan(); + } +} diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml index e2a801f06e069..c60a1f3129f01 100644 --- a/modules/indexing/pom.xml +++ b/modules/indexing/pom.xml @@ -40,6 +40,12 @@ ignite-core + + ${project.groupId} + ignite-spring + test + + commons-codec commons-codec diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java index 95ce6c1bf7e64..64f7efc26b3ab 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java @@ -121,6 +121,11 @@ public String plan() { return stmt.getPlanSQL(); } + /** */ + public long beginTs() { + return beginTs; + } + /** */ public long extWait() { return extWait; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 65600be66a71e..6851f158dcf98 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -106,6 +106,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.processors.query.running.HeavyQueriesTracker; import org.apache.ignite.internal.processors.query.running.RunningQueryManager; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.query.schema.AbstractSchemaChangeListener; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; @@ -455,6 +456,17 @@ private GridQueryFieldsResult executeSelectLocal( ); } + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + qryInfo.plan(), + qry, + qryDesc.schemaName(), + true, + qryInfo.beginTs(), + SqlPlanHistoryTracker.SqlEngine.H2 + ); + ResultSet rs = executeWithResumableTimeTracking( () -> executeSqlQueryWithTimer( stmt, @@ -2155,6 +2167,17 @@ private UpdateResult executeUpdate0( ) throws IgniteCheckedException { UpdatePlan plan = dml.plan(); + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + plan.plan(), + qryDesc.sql(), + qryDesc.schemaName(), + loc, + U.currentTimeMillis(), + SqlPlanHistoryTracker.SqlEngine.H2 + ); + UpdateResult fastUpdateRes = plan.processFast(qryParams.arguments()); if (fastUpdateRes != null) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java index e9ec4bd7a0df1..19f9f8d3fac5c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java @@ -57,7 +57,7 @@ private DmlArguments() { /** * Value argument. */ - private static class ConstantArgument implements DmlArgument { + public static class ConstantArgument implements DmlArgument { /** Value to return. */ private final Object val; @@ -74,6 +74,11 @@ private ConstantArgument(Object val) { @Override public Object get(Object[] params) { return val; } + + /** */ + public Object getValue() { + return val; + } } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java index 9ce39f071530b..dc367e2301e07 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java @@ -101,4 +101,19 @@ public UpdateResult execute(GridCacheAdapter cache, Object[] args) throws Ignite return res ? UpdateResult.ONE : UpdateResult.ZERO; } + + /** */ + public DmlArgument keyArg() { + return keyArg; + } + + /** */ + public DmlArgument valArg() { + return valArg; + } + + /** */ + public DmlArgument newValArg() { + return newValArg; + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index e68b71682e27b..e7402a1e8233b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -18,9 +18,12 @@ package org.apache.ignite.internal.processors.query.h2.dml; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; @@ -80,10 +83,10 @@ public final class UpdatePlan { /** Number of rows in rows based MERGE or INSERT. */ private final int rowsNum; - + /** Whether absent PK parts should be filled with defaults or not. */ private boolean fillAbsentPKsWithDefaults; - + /** Arguments for fast UPDATE or DELETE. */ private final FastUpdate fastUpdate; @@ -135,7 +138,7 @@ public UpdatePlan( this.rows = rows; this.rowsNum = rowsNum; this.fillAbsentPKsWithDefaults = fillAbsentPKsWithDefaults; - + assert mode != null; assert tbl != null; @@ -469,7 +472,7 @@ private void extractArgsValues(Object[] args, List> res, GridQueryRowDes colVal = row.size() > j ? row.get(j).get(args) : null; else colVal = row.get(j).get(args); - + if (j == keyColIdx || j == valColIdx) { Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); @@ -531,4 +534,84 @@ public boolean isLocalSubquery() { public boolean canSelectBeLazy() { return canSelectBeLazy; } + + /** + * @return String representation of the update plan. + */ + public String plan() { + StringBuilder sb = new StringBuilder("mode=" + mode + + "; table name=" + tbl.cacheName() + + "; group ID=" + tbl.cacheContext().groupId() + + "; columns=" + (colNames == null ? "N/A" : "[" + Arrays.stream(colNames).reduce((a, b) -> a + ", " + b) + .orElse("") + "]") + + "; column types=" + (colTypes == null ? "N/A" : "[" + Arrays.stream(colTypes).mapToObj(Objects::toString) + .reduce((a, b) -> a + ", " + b).orElse("") + "]") + + "; key column index=" + (keyColIdx < 0 ? "N/A" : keyColIdx) + + "; value column index=" + (valColIdx < 0 ? "N/A" : valColIdx) + + "; row count=" + rowsNum + + "; values=" + (rows == null ? "N/A" : "[" + rowsToString(rows) + "]") + + "; select statement based on initial DML statement=" + (selectQry == null ? "N/A" : "'" + selectQry + "'")); + + if (selectQry != null) { + sb.append(" (is actual subquery executed on cache=" + isLocSubqry + + ", can be executed in lazy mode=" + canSelectBeLazy + ")"); + } + + if (fastUpdate == null) + sb.append("; fast update=N/A"); + else { + List fastUpdateArgs = getFastUpdateArgs(); + + sb.append("; fast update arguments=[key=" + fastUpdateArgs.get(0) + ", value=" + fastUpdateArgs.get(1) + + ", new value=" + fastUpdateArgs.get(2) + "]"); + } + + sb.append("; distributed plan info="); + + if (distributed == null) + sb.append("N/A"); + else { + sb.append("[IDs of caches involved in update=[" + + distributed.getCacheIds().stream().map(Object::toString).collect(Collectors.joining(", ")) + "]" + + ", update involves only replicated caches=" + distributed.isReplicatedOnly()); + } + + return sb.toString(); + } + + /** + * @param rows Rows. + */ + public String rowsToString(List> rows) { + StringBuilder result = new StringBuilder(); + + for (List row : rows) { + if (!row.isEmpty()) { + result.append("{"); + + result.append(row.stream() + .map(arg -> ((DmlArguments.ConstantArgument)arg).getValue().toString()) + .collect(Collectors.joining(", "))); + + result.append("}, "); + } + } + + return result.length() > 0 ? result.substring(0, result.length() - 2) : ""; + } + + /** */ + public List getFastUpdateArgs() { + assert fastUpdate != null; + + DmlArguments.ConstantArgument keyArg = (DmlArguments.ConstantArgument)fastUpdate.keyArg(); + DmlArguments.ConstantArgument valArg = (DmlArguments.ConstantArgument)fastUpdate.valArg(); + DmlArguments.ConstantArgument newValArg = (DmlArguments.ConstantArgument)fastUpdate.newValArg(); + + return Arrays.asList( + keyArg.getValue() == null ? "NULL" : keyArg.getValue().toString(), + valArg.getValue() == null ? "NULL" : valArg.getValue().toString(), + newValArg.getValue() == null ? "NULL" : newValArg.getValue().toString() + ); + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 78a0e0aeb9657..47c7b949dee14 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -69,6 +69,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; import org.apache.ignite.internal.processors.tracing.Span; @@ -476,6 +477,17 @@ private void onQueryRequest0( ); } + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + qryInfo.plan(), + sql, + schemaName, + false, + qryInfo.beginTs(), + SqlPlanHistoryTracker.SqlEngine.H2 + ); + GridQueryCancel qryCancel = qryResults.queryCancel(qryIdx); ResultSet rs = h2.executeWithResumableTimeTracking( diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index cffa0dfd9b8e6..0cde8ca1b2c69 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -77,6 +77,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; import org.apache.ignite.internal.util.typedef.C2; @@ -526,6 +527,17 @@ else if (QueryUtils.wasCancelled(err)) ); } + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + qryInfo.plan(), + qry.originalSql(), + schemaName, + qry.isLocal(), + qryStartTime, + SqlPlanHistoryTracker.SqlEngine.H2 + ); + H2PooledConnection conn0 = conn; ResultSet res = h2.executeWithResumableTimeTracking( diff --git a/modules/indexing/src/test/config/plan-history-conf.xml b/modules/indexing/src/test/config/plan-history-conf.xml new file mode 100644 index 0000000000000..959be5045ff53 --- /dev/null +++ b/modules/indexing/src/test/config/plan-history-conf.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java index f0d0e22646799..118a67c7720c8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/metric/SqlViewExporterSpiTest.java @@ -460,7 +460,8 @@ public void testViews() { "DS_SETS", "DS_SEMAPHORES", "DS_QUEUES", - "PAGES_TIMESTAMP_HISTOGRAM" + "PAGES_TIMESTAMP_HISTOGRAM", + "SQL_PLANS_HISTORY" )); Set actViews = new TreeSet<>(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java new file mode 100644 index 0000000000000..04dd980221a91 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query; + +import org.apache.ignite.Ignite; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** Test for Sql plan history configuration. */ +public class SqlPlanHistoryConfigTest extends GridCommonAbstractTest { + /** Sql plan history size in the XML Spring config. */ + private static final int SQL_PLAN_HISTORY_SIZE_XML_CONFIG = 10; + + /** Checks that plan history size specified in XML config is respected. */ + @Test + public void testXmlConfigSqlPlanHistorySize() throws Exception { + String cfgPath = "modules/indexing/src/test/config/plan-history-conf.xml"; + + Ignite ignite = startGridsWithSpringCtx(2, false, cfgPath); + + assertEquals(SQL_PLAN_HISTORY_SIZE_XML_CONFIG, + ignite.configuration().getSqlConfiguration().getSqlPlanHistorySize()); + + stopAllGrids(); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java deleted file mode 100644 index 05506658aeddc..0000000000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.apache.ignite.internal.processors.query; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.SqlConfiguration; -import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -public class SqlPlanHistoryTest extends GridCommonAbstractTest { - - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName).setSqlConfiguration( - new SqlConfiguration().setQueryEnginesConfiguration(new IndexingQueryEngineConfiguration())); - } - - @Test - public void test() throws Exception { - IgniteEx ignite = startGrids(2); - - IgniteCache cache = ignite.getOrCreateCache( - new CacheConfiguration("test") - .setQueryEntities(Collections.singleton( - new QueryEntity() - .setTableName("test") - .setKeyType(Integer.class.getName()) - .setValueType(Integer.class.getName())))); - - for (int i = 0; i < 5; i++) - cache.put(i, i); - - Iterator> iter1 = cache.query( - new SqlFieldsQuery("SELECT * FROM test").setLocal(false)).iterator(); - - assertTrue(iter1.hasNext()); - } -} From 2e4f27cd9f8372ef7f8b7c51f9db569d9ca7ce9c Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 13:01:04 +1000 Subject: [PATCH 03/33] IGNITE-22557 SQL Plan History Configuration Test removed from ignite-indexing --- modules/indexing/pom.xml | 6 --- .../src/test/config/plan-history-conf.xml | 33 --------------- .../query/SqlPlanHistoryConfigTest.java | 41 ------------------- 3 files changed, 80 deletions(-) delete mode 100644 modules/indexing/src/test/config/plan-history-conf.xml delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml index c60a1f3129f01..e2a801f06e069 100644 --- a/modules/indexing/pom.xml +++ b/modules/indexing/pom.xml @@ -40,12 +40,6 @@ ignite-core - - ${project.groupId} - ignite-spring - test - - commons-codec commons-codec diff --git a/modules/indexing/src/test/config/plan-history-conf.xml b/modules/indexing/src/test/config/plan-history-conf.xml deleted file mode 100644 index 959be5045ff53..0000000000000 --- a/modules/indexing/src/test/config/plan-history-conf.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - - - - diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java deleted file mode 100644 index 04dd980221a91..0000000000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPlanHistoryConfigTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query; - -import org.apache.ignite.Ignite; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** Test for Sql plan history configuration. */ -public class SqlPlanHistoryConfigTest extends GridCommonAbstractTest { - /** Sql plan history size in the XML Spring config. */ - private static final int SQL_PLAN_HISTORY_SIZE_XML_CONFIG = 10; - - /** Checks that plan history size specified in XML config is respected. */ - @Test - public void testXmlConfigSqlPlanHistorySize() throws Exception { - String cfgPath = "modules/indexing/src/test/config/plan-history-conf.xml"; - - Ignite ignite = startGridsWithSpringCtx(2, false, cfgPath); - - assertEquals(SQL_PLAN_HISTORY_SIZE_XML_CONFIG, - ignite.configuration().getSqlConfiguration().getSqlPlanHistorySize()); - - stopAllGrids(); - } -} From 95ab41bf28deb97aa928ae404a83a5ce2542a433 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 13:15:06 +1000 Subject: [PATCH 04/33] IGNITE-22557 Tests added to IgniteCalciteTestSuite --- .../ignite/testsuites/IgniteCalciteTestSuite.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java index e71990733db80..fa264f383a949 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java @@ -18,6 +18,11 @@ package org.apache.ignite.testsuites; import org.apache.ignite.internal.processors.query.calcite.QueryCheckerTest; +import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryCalciteFromClientSelfTest; +import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryCalciteSelfTest; +import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryConfigTest; +import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryH2FromClientSelfTest; +import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryH2SelfTest; import org.apache.ignite.internal.processors.query.calcite.exec.ClosableIteratorsHolderTest; import org.apache.ignite.internal.processors.query.calcite.exec.LogicalRelImplementorTest; import org.apache.ignite.internal.processors.query.calcite.exec.NumericTypesPrecisionsTest; @@ -50,6 +55,12 @@ CalciteCommunicationMessageSerializationTest.class, NumericTypesPrecisionsTest.class, + + SqlPlanHistoryConfigTest.class, + SqlPlanHistoryH2SelfTest.class, + SqlPlanHistoryH2FromClientSelfTest.class, + SqlPlanHistoryCalciteSelfTest.class, + SqlPlanHistoryCalciteFromClientSelfTest.class, }) public class IgniteCalciteTestSuite { } From 318ffdfbccc76ab68539cabb2178a8f4b65a3fe4 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 14:28:00 +1000 Subject: [PATCH 05/33] IGNITE-22557 Tests minor refactoring --- ...lPlanHistoryCalciteFromClientSelfTest.java | 7 + .../SqlPlanHistoryCalciteSelfTest.java | 8 + .../SqlPlanHistoryH2FromClientSelfTest.java | 7 + .../calcite/SqlPlanHistoryH2SelfTest.java | 143 +++++++++--------- .../query/running/SqlPlanHistoryTracker.java | 2 +- 5 files changed, 93 insertions(+), 74 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java index 9b17a65c458f4..b51bd7c66b231 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java @@ -22,6 +22,13 @@ /** Tests for SQL plan history from client (Calcite engine). */ public class SqlPlanHistoryCalciteFromClientSelfTest extends SqlPlanHistoryCalciteSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + setClientMode(true); + } + /** {@inheritDoc} */ @Override protected IgniteEx queryNode() { IgniteEx node = grid(1); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java index 52940c6ca0c8f..66c428c7d7d34 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; +import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; /** Tests for SQL plan history (Calcite engine). */ public class SqlPlanHistoryCalciteSelfTest extends SqlPlanHistoryH2SelfTest { @@ -26,4 +27,11 @@ public class SqlPlanHistoryCalciteSelfTest extends SqlPlanHistoryH2SelfTest { @Override protected QueryEngineConfigurationEx configureSqlEngine() { return new CalciteQueryEngineConfiguration(); } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + setSqlEngine(SqlPlanHistoryTracker.SqlEngine.CALCITE); + } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java index a5ae70e8413ed..9a79114f94e29 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java @@ -22,6 +22,13 @@ /** Tests for SQL plan history from client (H2 engine). */ public class SqlPlanHistoryH2FromClientSelfTest extends SqlPlanHistoryH2SelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + setClientMode(true); + } + /** {@inheritDoc} */ @Override protected IgniteEx queryNode() { IgniteEx node = grid(1); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java index 495378118441d..b10c520f75490 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java @@ -35,7 +35,6 @@ import org.apache.ignite.cache.query.TextQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.cache.query.annotations.QuerySqlFunction; -import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.SqlConfiguration; @@ -56,31 +55,31 @@ /** Tests for SQL plan history (H2 engine). */ @RunWith(Parameterized.class) public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { - /** SQL Plan history size. */ + /** SQL plan history size. */ private static final int PLAN_HISTORY_SIZE = 10; /** Plan history size excess. */ private static final int PLAN_HISTORY_EXCESS = 2; - /** Simple query. */ + /** Simple SQL query. */ private static final String SQL = "SELECT * FROM A.String"; - /** Failed query. */ + /** Failed SQL query. */ private static final String SQL_FAILED = "select * from A.String where A.fail()=1"; - /** Cross-cache query. */ + /** Cross-cache SQL query. */ private static final String SQL_CROSS_CACHE = "SELECT * FROM B.String"; - /** Failed cross-cache query. */ + /** Failed cross-cache SQL query. */ private static final String SQL_CROSS_CACHE_FAILED = "select * from A.String where A.fail()=1"; - /** Query with reduce phase. */ + /** SQL query with reduce phase. */ private static final String SQL_WITH_REDUCE_PHASE = "select o.name n1, p.name n2 from \"pers\".Person p, " + "\"org\".Organization o where p.orgId=o._key and o._key=101" + " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + " where p.orgId=o._key and o._key=102"; - /** Sets of DML commands of various complexity including failed commands. */ + /** Sets of DML commands of various complexity (including failed commands). */ List> dmlCmds = Arrays.asList( Arrays.asList( "insert into A.String (_key, _val) values(101, '101')", @@ -115,51 +114,51 @@ public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { /** Strings for checking SQL plan history after executing DML commands. */ List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); - /** Successful SqlFields query. */ + /** Successful SqlFieldsQuery. */ SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); - /** Failed SqlFields query. */ + /** Failed SqlFieldsQuery. */ SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); - /** Successful cross-cache SqlFields query. */ + /** Successful cross-cache SqlFieldsQuery. */ SqlFieldsQuery sqlFieldsCrossCacheQry = new SqlFieldsQuery(SQL_CROSS_CACHE); - /** Failed cross-cache SqlFields query. */ + /** Failed cross-cache SqlFieldsQuery. */ SqlFieldsQuery sqlFieldsCrossCacheQryFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); - /** Successful Sql query. */ - SqlQuery sqlQry = new SqlQuery<>("String", "from String"); - - /** Failed Sql query. */ - SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); - - /** Scan query. */ - ScanQuery scanQry = new ScanQuery<>(); - - /** Text query. */ - TextQuery textQry = new TextQuery<>("String", "2"); - - /** */ + /** Successful SqlFieldsQuery with reduce phase. */ private final SqlFieldsQuery sqlFieldsQryWithReducePhase = new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE) .setDistributedJoins(true); - /** */ + /** Failed SqlFieldsQueries with reduce phase. */ private final SqlFieldsQuery[] sqlFieldsQryWithReducePhaseFailed = F.asArray( new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=101", "fail()")).setDistributedJoins(true), new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=102", "fail()")).setDistributedJoins(true) ); + /** Successful SqlQuery. */ + SqlQuery sqlQry = new SqlQuery<>("String", "from String"); + + /** Failed SqlQuery. */ + SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); + + /** ScanQuery. */ + ScanQuery scanQry = new ScanQuery<>(); + + /** TextQuery. */ + TextQuery textQry = new TextQuery<>("String", "2"); + /** Client mode flag. */ private boolean isClient; - /** Sql engine. */ - private SqlPlanHistoryTracker.SqlEngine sqlEngine; + /** SQL engine. */ + protected SqlPlanHistoryTracker.SqlEngine sqlEngine = SqlPlanHistoryTracker.SqlEngine.H2; /** Local query flag. */ @Parameterized.Parameter public boolean loc; - /** Fully fetched query flag. */ + /** Fully-fetched query flag. */ @Parameterized.Parameter(1) public boolean isFullyFetched; @@ -208,6 +207,26 @@ private CacheConfiguration configureCahce(String name, Class... idxTypes) { .setSqlFunctionClasses(Functions.class); } + /** + * @return Ignite instance for quering. + */ + protected IgniteEx queryNode() { + IgniteEx node = grid(0); + + assertFalse(node.context().clientNode()); + + return node; + } + + /** + * Starts Ignite instance. + * + * @throws Exception In case of failure. + */ + protected void startTestGrid() throws Exception { + startGrid(0); + } + /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { super.beforeTestsStarted(); @@ -242,22 +261,20 @@ private CacheConfiguration configureCahce(String name, Class... idxTypes) { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { resetPlanHistory(); + } - sqlEngine = isCalcite() ? SqlPlanHistoryTracker.SqlEngine.CALCITE : SqlPlanHistoryTracker.SqlEngine.H2; - - try { - if (grid(1).configuration().isClientMode()) - isClient = true; - } - catch (Exception ignore) { - //No-Op - } + /** + * @param sqlEngine Sql engine. + */ + protected void setSqlEngine(SqlPlanHistoryTracker.SqlEngine sqlEngine) { + this.sqlEngine = sqlEngine; } - /** */ - public boolean isCalcite() { - return grid(0).context().config().getSqlConfiguration().getQueryEnginesConfiguration()[0] instanceof - CalciteQueryEngineConfiguration; + /** + * @param isClient Client more flag. + */ + protected void setClientMode(boolean isClient) { + this.isClient = isClient; } /** */ @@ -294,31 +311,31 @@ public void testJdbcQueryFailed() { checkSqlPlanHistory(getExpectedHistorySize()); } - /** Checks successful SqlFields queries. */ + /** Checks successful SqlFieldsQuery. */ @Test public void testSqlFieldsQuery() { executeSuccessfulQuery(sqlFieldsQry); } - /** Checks failed SqlFields queries. */ + /** Checks failed SqlFieldsQuery. */ @Test public void testSqlFieldsQueryFailed() { executeFailedQuery(sqlFieldsQryFailed); } - /** Checks successful cross-cache SqlFields queries. */ + /** Checks successful cross-cache SqlFieldsQuery. */ @Test public void testSqlFieldsCrossCacheQuery() { executeSuccessfulQuery(sqlFieldsCrossCacheQry); } - /** Checks failed cross-cache SqlFields queries. */ + /** Checks failed cross-cache SqlFieldsQuery. */ @Test public void testSqlFieldsCrossCacheQueryFailed() { executeFailedQuery(sqlFieldsCrossCacheQryFailed); } - /** Checks successful SqlFields queries with reduce phase. */ + /** Checks successful SqlFieldsQuery with reduce phase. */ @Test public void testSqlFieldsQueryWithReducePhase() { if (loc) @@ -329,7 +346,7 @@ public void testSqlFieldsQueryWithReducePhase() { checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); } - /** Checks failed SqlFields queries with reduce phase. */ + /** Checks failed SqlFieldsQuery with reduce phase. */ @Test public void testSqlFieldsQueryWithReducePhaseFailed() { if (loc) @@ -349,25 +366,25 @@ public void testSqlFieldsQueryWithReducePhaseFailed() { } } - /** Checks successful Sql queries. */ + /** Checks successful SqlQuery. */ @Test public void testSqlQuery() { executeSuccessfulQuery(sqlQry); } - /** Checks failed Sql queries. */ + /** Checks failed SqlQuery. */ @Test public void testSqlQueryFailed() { executeFailedQuery(sqlQryFailed); } - /** Checks scan queries. */ + /** Checks ScanQuery. */ @Test public void testScanQuery() { executeQueryWithoutPlan(scanQry); } - /** Checks text queries. */ + /** Checks TextQuery. */ @Test public void testTextQuery() { executeQueryWithoutPlan(textQry); @@ -400,7 +417,7 @@ public void testJdbcDml() throws Exception { } } - /** Checks DML commands executed via SqlField queries. */ + /** Checks DML commands executed via SqlFieldsQuery. */ @Test public void testSqlFieldsDml() { if (isClient && loc || !isFullyFetched) @@ -613,26 +630,6 @@ public void checkMetrics(int size, Collection sqlPlans) { } } - /** - * @return Ignite instance for quering. - */ - protected IgniteEx queryNode() { - IgniteEx node = grid(0); - - assertFalse(node.context().clientNode()); - - return node; - } - - /** - * Starts Ignite instance. - * - * @throws Exception In case of failure. - */ - protected void startTestGrid() throws Exception { - startGrid(0); - } - /** */ private static class Person { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index 37018a55ae74d..6e0478aea58f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; -/** */ +/** Class that manages recording and storing SQL plans. */ public class SqlPlanHistoryTracker { /** SQL plan history. */ private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; From bfd9fe468d3bba0838617a7ee5c00113ef5461fa Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 14:35:23 +1000 Subject: [PATCH 06/33] IGNITE-22557 Minor refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index e7402a1e8233b..8103bb52b5606 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -536,7 +536,7 @@ public boolean canSelectBeLazy() { } /** - * @return String representation of the update plan. + * @return String representation of update plan. */ public String plan() { StringBuilder sb = new StringBuilder("mode=" + mode + From 21b5807101aac8e9efdc4d1beb7143ec9877cec6 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 14:36:53 +1000 Subject: [PATCH 07/33] IGNITE-22557 Minor refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 8103bb52b5606..e7402a1e8233b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -536,7 +536,7 @@ public boolean canSelectBeLazy() { } /** - * @return String representation of update plan. + * @return String representation of the update plan. */ public String plan() { StringBuilder sb = new StringBuilder("mode=" + mode + From 40f22913390ff2345f3131f6309a4cfb9099418c Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 14:39:04 +1000 Subject: [PATCH 08/33] IGNITE-22557 Minor refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index e7402a1e8233b..8103bb52b5606 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -536,7 +536,7 @@ public boolean canSelectBeLazy() { } /** - * @return String representation of the update plan. + * @return String representation of update plan. */ public String plan() { StringBuilder sb = new StringBuilder("mode=" + mode + From adeefb5a1e79c601482c67272ca4f012a3182266 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 15:16:18 +1000 Subject: [PATCH 09/33] IGNITE-22557 Minor UpdatePlan#plan refactoring --- .../internal/processors/query/h2/dml/UpdatePlan.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 8103bb52b5606..4324ccecd708f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -550,12 +550,9 @@ public String plan() { "; value column index=" + (valColIdx < 0 ? "N/A" : valColIdx) + "; row count=" + rowsNum + "; values=" + (rows == null ? "N/A" : "[" + rowsToString(rows) + "]") + - "; select statement based on initial DML statement=" + (selectQry == null ? "N/A" : "'" + selectQry + "'")); - - if (selectQry != null) { - sb.append(" (is actual subquery executed on cache=" + isLocSubqry + - ", can be executed in lazy mode=" + canSelectBeLazy + ")"); - } + "; select statement based on initial DML statement=" + (selectQry == null ? "N/A" : + ("'" + selectQry + "' (is actual subquery executed on cache=" + isLocSubqry + + ", can be executed in lazy mode=" + canSelectBeLazy + ")"))); if (fastUpdate == null) sb.append("; fast update=N/A"); From 63831eb23d76e740d93538453042ee66ba56822e Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Fri, 2 Aug 2024 15:22:42 +1000 Subject: [PATCH 10/33] IGNITE-22557 Sequence of columns in the view changed --- .../walker/SqlPlanHistoryViewWalker.java | 18 +++++++-------- .../systemview/view/SqlPlanHistoryView.java | 22 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java index b9825180db2de..2bf49f90b627b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/systemview/walker/SqlPlanHistoryViewWalker.java @@ -24,7 +24,7 @@ /** * Generated by {@code org.apache.ignite.codegen.SystemViewRowAttributeWalkerGenerator}. * {@link SqlPlanHistoryView} attributes walker. - * + * * @see SqlPlanHistoryView */ public class SqlPlanHistoryViewWalker implements SystemViewRowAttributeWalker { @@ -32,20 +32,20 @@ public class SqlPlanHistoryViewWalker implements SystemViewRowAttributeWalker Date: Mon, 5 Aug 2024 11:02:30 +1000 Subject: [PATCH 11/33] IGNITE-22557 SqlPlanHistoryH2SelfTest refactoring --- .../calcite/SqlPlanHistoryH2SelfTest.java | 259 +++++++++++------- 1 file changed, 164 insertions(+), 95 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java index b10c520f75490..19ec8ed069aa0 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.query.Query; @@ -79,52 +80,56 @@ public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + " where p.orgId=o._key and o._key=102"; - /** Sets of DML commands of various complexity (including failed commands). */ - List> dmlCmds = Arrays.asList( - Arrays.asList( - "insert into A.String (_key, _val) values(101, '101')", - "update A.String set _val='111' where _key=101", - "delete from A.String where _key=101" - ), - Arrays.asList( - "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", - "update A.String set _val = case _key " + - "when 101 then '111' " + - "when 102 then '112' " + - "when 103 then '113' " + - "end " + - "where _key in (101, 102, 103)", - "delete from A.String where _key in (101, 102, 103)" - ), - Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name " + - "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", - "update A.String set _val = 'updated' where _key in " + - "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", - "delete from A.String where _key in (select orgId from \"pers\".Person)" - ), - Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + - "where A.fail()=1", - "update A.String set _val = 'failed' where A.fail()=1", - "delete from A.String where A.fail()=1" - ) + /** Set of simple DML commands. */ + private final List dmlCmds = Arrays.asList( + "insert into A.String (_key, _val) values(101, '101')", + "update A.String set _val='111' where _key=101", + "delete from A.String where _key=101" + ); + + /** Set of DML commands with multiple operations. */ + private final List dmlCmdsMultiOps = Arrays.asList( + "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", + "update A.String set _val = case _key " + + "when 101 then '111' " + + "when 102 then '112' " + + "when 103 then '113' " + + "end " + + "where _key in (101, 102, 103)", + "delete from A.String where _key in (101, 102, 103)" + ); + + /** Set of DML commands with joins. */ + private final List dmlCmdsWithJoins = Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name " + + "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", + "update A.String set _val = 'updated' where _key in " + + "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", + "delete from A.String where _key in (select orgId from \"pers\".Person)" + ); + + /** Set of failed DML commands. */ + private final List dmlCmdsFailed = Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + + "where A.fail()=1", + "update A.String set _val = 'failed' where A.fail()=1", + "delete from A.String where A.fail()=1" ); /** Strings for checking SQL plan history after executing DML commands. */ - List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); + private final List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); /** Successful SqlFieldsQuery. */ - SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); + private final SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); /** Failed SqlFieldsQuery. */ - SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); + private final SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); /** Successful cross-cache SqlFieldsQuery. */ - SqlFieldsQuery sqlFieldsCrossCacheQry = new SqlFieldsQuery(SQL_CROSS_CACHE); + private final SqlFieldsQuery sqlFieldsCrossCacheQry = new SqlFieldsQuery(SQL_CROSS_CACHE); /** Failed cross-cache SqlFieldsQuery. */ - SqlFieldsQuery sqlFieldsCrossCacheQryFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); + private final SqlFieldsQuery sqlFieldsCrossCacheQryFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); /** Successful SqlFieldsQuery with reduce phase. */ private final SqlFieldsQuery sqlFieldsQryWithReducePhase = new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE) @@ -137,16 +142,16 @@ public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { ); /** Successful SqlQuery. */ - SqlQuery sqlQry = new SqlQuery<>("String", "from String"); + private final SqlQuery sqlQry = new SqlQuery<>("String", "from String"); /** Failed SqlQuery. */ - SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); + private final SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); /** ScanQuery. */ - ScanQuery scanQry = new ScanQuery<>(); + private final ScanQuery scanQry = new ScanQuery<>(); /** TextQuery. */ - TextQuery textQry = new TextQuery<>("String", "2"); + private final TextQuery textQry = new TextQuery<>("String", "2"); /** Client mode flag. */ private boolean isClient; @@ -289,7 +294,7 @@ public void testJdbcQuery() throws SQLException { return; for (int i = 0; i < 2; i++) { - runJdbcQuery(SQL); + jdbcQuery(SQL); checkSqlPlanHistory(getExpectedHistorySize()); } @@ -302,7 +307,7 @@ public void testJdbcQueryFailed() { return; try { - runJdbcQuery(SQL_FAILED); + jdbcQuery(SQL_FAILED); } catch (Exception ignore) { //No-Op @@ -314,25 +319,25 @@ public void testJdbcQueryFailed() { /** Checks successful SqlFieldsQuery. */ @Test public void testSqlFieldsQuery() { - executeSuccessfulQuery(sqlFieldsQry); + runSuccessfulQuery(sqlFieldsQry); } /** Checks failed SqlFieldsQuery. */ @Test public void testSqlFieldsQueryFailed() { - executeFailedQuery(sqlFieldsQryFailed); + runFailedQuery(sqlFieldsQryFailed); } /** Checks successful cross-cache SqlFieldsQuery. */ @Test public void testSqlFieldsCrossCacheQuery() { - executeSuccessfulQuery(sqlFieldsCrossCacheQry); + runSuccessfulQuery(sqlFieldsCrossCacheQry); } /** Checks failed cross-cache SqlFieldsQuery. */ @Test public void testSqlFieldsCrossCacheQueryFailed() { - executeFailedQuery(sqlFieldsCrossCacheQryFailed); + runFailedQuery(sqlFieldsCrossCacheQryFailed); } /** Checks successful SqlFieldsQuery with reduce phase. */ @@ -341,7 +346,7 @@ public void testSqlFieldsQueryWithReducePhase() { if (loc) return; - runCacheQuery(sqlFieldsQryWithReducePhase, "pers"); + cacheQuery(sqlFieldsQryWithReducePhase, "pers"); checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); } @@ -354,7 +359,7 @@ public void testSqlFieldsQueryWithReducePhaseFailed() { for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { try { - runCacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); + cacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); } catch (Exception ignore) { //No-Op @@ -369,67 +374,90 @@ public void testSqlFieldsQueryWithReducePhaseFailed() { /** Checks successful SqlQuery. */ @Test public void testSqlQuery() { - executeSuccessfulQuery(sqlQry); + runSuccessfulQuery(sqlQry); } /** Checks failed SqlQuery. */ @Test public void testSqlQueryFailed() { - executeFailedQuery(sqlQryFailed); + runFailedQuery(sqlQryFailed); } /** Checks ScanQuery. */ @Test public void testScanQuery() { - executeQueryWithoutPlan(scanQry); + runQueryWithoutPlan(scanQry); } /** Checks TextQuery. */ @Test public void testTextQuery() { - executeQueryWithoutPlan(textQry); + runQueryWithoutPlan(textQry); } /** Checks DML commands executed via JDBC. */ @Test - public void testJdbcDml() throws Exception { - if (loc || !isFullyFetched) - return; + public void testJdbcDml() throws SQLException { + runJdbcDml(dmlCmds); + } - try ( - Connection conn = GridTestUtils.connect(queryNode(), null); - Statement stmt = conn.createStatement() - ) { - for (List setOfCmds : dmlCmds) { - for (String cmd : setOfCmds) { - try { - stmt.execute(cmd); - } - catch (Exception ignore) { - //No-Op - } - } + /** Checks DML commands with multiple operations executed via JDBC. */ + @Test + public void testJdbcDmlMultiOps() throws SQLException { + runJdbcDml(dmlCmdsMultiOps); + } - checkSqlPlanHistoryDml(3); + /** Checks DML commands with joins executed via JDBC. */ + @Test + public void testJdbcDmlWithJoins() throws SQLException { + runJdbcDml(dmlCmdsWithJoins); + } - resetPlanHistory(); - } - } + /** Checks failed DML commands executed via JDBC. */ + @Test + public void testJdbcDmlFailed() throws SQLException { + executeJdbcDml((stmt) -> { + for (String cmd : dmlCmdsFailed) + try { + stmt.execute(cmd); + } + catch (Exception ignore) { + //No-Op + } + }); } /** Checks DML commands executed via SqlFieldsQuery. */ @Test public void testSqlFieldsDml() { - if (isClient && loc || !isFullyFetched) - return; + runSqlFieldsQueryDml(dmlCmds); + } - for (List setOfCmds : dmlCmds) { - setOfCmds.forEach(this::runDmlCommand); + /** Checks DML commands with multiple operations executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlMultiOps() { + runSqlFieldsQueryDml(dmlCmdsMultiOps); + } - checkSqlPlanHistoryDml(3); + /** Checks DML commands with joins executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlWithJoins() { + runSqlFieldsQueryDml(dmlCmdsWithJoins); + } - resetPlanHistory(); - } + /** Checks failed DML commands executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlFailed() { + executeSqlFieldsQueryDml(cache -> { + for (String cmd : dmlCmdsFailed) { + try { + cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); + } + catch (Exception ignore) { + //No-Op + } + } + }); } /** Checks that older plan entries are evicted when maximum history size is reached. */ @@ -440,7 +468,7 @@ public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { try { - runCacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); + cacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); } catch (Exception ignore) { //No-Op @@ -460,10 +488,10 @@ public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { /** * @param qry Query. */ - public void executeSuccessfulQuery(Query qry) { + public void runSuccessfulQuery(Query qry) { executeQuery(qry, () -> { for (int i = 0; i < 2; i++) { - runCacheQuery(qry, "A"); + cacheQuery(qry, "A"); checkSqlPlanHistory(getExpectedHistorySize()); } @@ -473,10 +501,10 @@ public void executeSuccessfulQuery(Query qry) { /** * @param qry Query. */ - public void executeFailedQuery(Query qry) { + public void runFailedQuery(Query qry) { executeQuery(qry, () -> { try { - runCacheQuery(qry, "A"); + cacheQuery(qry, "A"); } catch (Exception ignore) { //No-Op @@ -489,9 +517,9 @@ public void executeFailedQuery(Query qry) { /** * @param qry Query. */ - public void executeQueryWithoutPlan(Query qry) { + public void runQueryWithoutPlan(Query qry) { executeQuery(qry, () -> { - runCacheQuery(qry, "A"); + cacheQuery(qry, "A"); checkSqlPlanHistory(0); }); @@ -513,7 +541,7 @@ public void executeQuery(Query qry, Runnable task) { /** * @param qry Query. */ - private void runJdbcQuery(String qry) throws SQLException { + private void jdbcQuery(String qry) throws SQLException { try ( Connection conn = GridTestUtils.connect(queryNode(), null); Statement stmt = conn.createStatement() @@ -531,7 +559,7 @@ private void runJdbcQuery(String qry) throws SQLException { * @param qry Query. * @param cacheName Cache name. */ - public void runCacheQuery(Query qry, String cacheName) { + public void cacheQuery(Query qry, String cacheName) { IgniteCache cache = queryNode().getOrCreateCache(cacheName); if (isFullyFetched) @@ -546,17 +574,58 @@ public void runCacheQuery(Query qry, String cacheName) { } /** - * @param cmd DML command. + * @param cmds Set of DML commands. */ - public void runDmlCommand(String cmd) { - IgniteCache cache = queryNode().getOrCreateCache("A"); + public void runJdbcDml(List cmds) throws SQLException { + executeJdbcDml((stmt) -> { + for (String cmd : cmds) { + try { + stmt.execute(cmd); + } + catch (SQLException e) { + throw new RuntimeException(e); + } + } + }); + } - try { - cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); - } - catch (Exception ignore) { - //No-Op + /** + * @param task Task to execute. + */ + public void executeJdbcDml(Consumer task) throws SQLException { + if (loc || !isFullyFetched) + return; + + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + task.accept(stmt); } + + checkSqlPlanHistoryDml(3); + } + + /** + * @param cmds Set of DML commands. + */ + public void runSqlFieldsQueryDml(List cmds) { + executeSqlFieldsQueryDml(cache -> + cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc)))); + } + + /** + * @param task Task to execute. + */ + public void executeSqlFieldsQueryDml(Consumer> task) { + if (isClient && loc || !isFullyFetched) + return; + + IgniteCache cache = queryNode().getOrCreateCache("A"); + + task.accept(cache); + + checkSqlPlanHistoryDml(3); } /** */ From 27f46abc78db1273eff385bec96732ddb51e2870 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Mon, 5 Aug 2024 12:06:38 +1000 Subject: [PATCH 12/33] IGNITE-22557 Description of the view added to system-views.adoc --- docs/_docs/monitoring-metrics/system-views.adoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/_docs/monitoring-metrics/system-views.adoc b/docs/_docs/monitoring-metrics/system-views.adoc index 915ad69459c9f..060b2308598cf 100644 --- a/docs/_docs/monitoring-metrics/system-views.adoc +++ b/docs/_docs/monitoring-metrics/system-views.adoc @@ -519,6 +519,21 @@ This view exposes information about currently running SQL queries. |LAST_START_TIME | date | Last execution date |=== +== SQL_PLANS_HISTORY + +This view displays a list of plans of previously executed SQL queries. + +[{table_opts}] +|=== +|NAME | TYPE | DESCRIPTION +|SCHEMA_NAME | string | Schema name +|SQL | string | Query text +|PLAN | string | SQL plan text +|LOCAL | boolean | True if local only +|ENGINE | string | SQL engine +|LAST_START_TIME | date | Last execution date +|=== + == SCHEMAS This view exposes information about SQL schemas. From a338ccdc6b91c47af30811e3c5823770f99643c9 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Mon, 5 Aug 2024 15:50:37 +1000 Subject: [PATCH 13/33] IGNITE-22557 Tests refactoring, SqlPlanHistoryCalciteSelfTest#testEmptyPlanHistory added, minor Javadoc changes --- .../SqlPlanHistoryCalciteSelfTest.java | 733 +++++++++++++++++- .../calcite/SqlPlanHistoryH2SelfTest.java | 715 +---------------- .../testsuites/IgniteCalciteTestSuite.java | 4 +- .../configuration/SqlConfiguration.java | 2 +- .../query/running/RunningQueryManager.java | 4 +- .../processors/query/running/SqlPlan.java | 2 +- .../query/running/SqlPlanHistoryTracker.java | 9 +- 7 files changed, 747 insertions(+), 722 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java index 66c428c7d7d34..281288c52242e 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java @@ -17,21 +17,746 @@ package org.apache.ignite.internal.processors.query.calcite; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QuerySqlFunction; import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; +import org.apache.ignite.internal.processors.query.running.SqlPlan; import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; /** Tests for SQL plan history (Calcite engine). */ -public class SqlPlanHistoryCalciteSelfTest extends SqlPlanHistoryH2SelfTest { +@RunWith(Parameterized.class) +public class SqlPlanHistoryCalciteSelfTest extends GridCommonAbstractTest { + /** SQL plan history size. */ + private static final int PLAN_HISTORY_SIZE = 10; + + /** SQL plan history size excess. */ + private static final int PLAN_HISTORY_EXCESS = 2; + + /** Simple SQL query. */ + private static final String SQL = "SELECT * FROM A.String"; + + /** Failed SQL query. */ + private static final String SQL_FAILED = "select * from A.String where A.fail()=1"; + + /** Cross-cache SQL query. */ + private static final String SQL_CROSS_CACHE = "SELECT * FROM B.String"; + + /** Failed cross-cache SQL query. */ + private static final String SQL_CROSS_CACHE_FAILED = "select * from B.String where B.fail()=1"; + + /** SQL query with reduce phase. */ + private static final String SQL_WITH_REDUCE_PHASE = "select o.name n1, p.name n2 from \"pers\".Person p, " + + "\"org\".Organization o where p.orgId=o._key and o._key=101" + + " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + + " where p.orgId=o._key and o._key=102"; + + /** Set of simple DML commands. */ + private final List dmlCmds = Arrays.asList( + "insert into A.String (_key, _val) values(101, '101')", + "update A.String set _val='111' where _key=101", + "delete from A.String where _key=101" + ); + + /** Set of DML commands with multiple operations. */ + private final List dmlCmdsMultiOps = Arrays.asList( + "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", + "update A.String set _val = case _key " + + "when 101 then '111' " + + "when 102 then '112' " + + "when 103 then '113' " + + "end " + + "where _key in (101, 102, 103)", + "delete from A.String where _key in (101, 102, 103)" + ); + + /** Set of DML commands with joins. */ + private final List dmlCmdsWithJoins = Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name " + + "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", + "update A.String set _val = 'updated' where _key in " + + "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", + "delete from A.String where _key in (select orgId from \"pers\".Person)" + ); + + /** Set of failed DML commands. */ + private final List dmlCmdsFailed = Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + + "where A.fail()=1", + "update A.String set _val = 'failed' where A.fail()=1", + "delete from A.String where A.fail()=1" + ); + + /** Strings for checking SQL plan history after executing DML commands. */ + private final List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); + + /** Successful SqlFieldsQuery. */ + private final SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); + + /** Failed SqlFieldsQuery. */ + private final SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); + + /** Successful cross-cache SqlFieldsQuery. */ + private final SqlFieldsQuery sqlFieldsQryCrossCache = new SqlFieldsQuery(SQL_CROSS_CACHE); + + /** Failed cross-cache SqlFieldsQuery. */ + private final SqlFieldsQuery sqlFieldsQryCrossCacheFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); + + /** Successful SqlFieldsQuery with reduce phase. */ + private final SqlFieldsQuery sqlFieldsQryWithReducePhase = new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE) + .setDistributedJoins(true); + + /** Failed SqlFieldsQueries with reduce phase. */ + private final SqlFieldsQuery[] sqlFieldsQryWithReducePhaseFailed = F.asArray( + new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=101", "fail()")), + new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=102", "fail()")) + ); + + /** Successful SqlQuery. */ + private final SqlQuery sqlQry = new SqlQuery<>("String", "from String"); + + /** Failed SqlQuery. */ + private final SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); + + /** ScanQuery. */ + private final ScanQuery scanQry = new ScanQuery<>(); + + /** TextQuery. */ + private final TextQuery textQry = new TextQuery<>("String", "2"); + + /** Client mode flag. */ + private boolean isClient; + + /** SQL engine. */ + protected SqlPlanHistoryTracker.SqlEngine sqlEngine = SqlPlanHistoryTracker.SqlEngine.CALCITE; + + /** Local query flag. */ + @Parameterized.Parameter + public boolean loc; + + /** Fully-fetched query flag. */ + @Parameterized.Parameter(1) + public boolean isFullyFetched; + + /** */ + @Parameterized.Parameters(name = "loc={0}, fullyFetched={1}") + public static Collection params() { + return Arrays.asList(new Object[][] { + {true, true}, {true, false}, {false, true}, {false, false} + }); + } + /** {@inheritDoc} */ - @Override protected QueryEngineConfigurationEx configureSqlEngine() { + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + QueryEngineConfigurationEx engCfg = configureSqlEngine(); + + cfg.setSqlConfiguration(new SqlConfiguration() + .setSqlPlanHistorySize(PLAN_HISTORY_SIZE) + .setQueryEnginesConfiguration(engCfg) + ); + + return cfg.setCacheConfiguration( + configureCahce("A", Integer.class, String.class), + configureCahce("B", Integer.class, String.class), + configureCahce("pers", Integer.class, Person.class), + configureCahce("org", Integer.class, Organization.class) + ); + } + + /** */ + protected QueryEngineConfigurationEx configureSqlEngine() { return new CalciteQueryEngineConfiguration(); } + /** + * @param name Name. + * @param idxTypes Index types. + * @return Cache configuration. + */ + @SuppressWarnings("unchecked") + private CacheConfiguration configureCahce(String name, Class... idxTypes) { + return new CacheConfiguration() + .setName(name) + .setIndexedTypes(idxTypes) + .setSqlFunctionClasses(Functions.class); + } + + /** + * @return Ignite node where queries are executed. + */ + protected IgniteEx queryNode() { + IgniteEx node = grid(0); + + assertFalse(node.context().clientNode()); + + return node; + } + + /** + * Starts Ignite instance. + * + * @throws Exception In case of failure. + */ + protected void startTestGrid() throws Exception { + startGrid(0); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startTestGrid(); + + IgniteCache cacheA = queryNode().cache("A"); + IgniteCache cacheB = queryNode().cache("B"); + + for (int i = 0; i < 100; i++) { + cacheA.put(i, String.valueOf(i)); + cacheB.put(i, String.valueOf(i)); + } + + IgniteCache cachePers = queryNode().cache("pers"); + IgniteCache cacheOrg = queryNode().cache("org"); + + cacheOrg.put(101, new Organization("o1")); + cacheOrg.put(102, new Organization("o2")); + cachePers.put(103, new Person(101, "p1")); + cachePers.put(104, new Person(102, "p2")); + cachePers.put(105, new Person(103, "p3")); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + stopAllGrids(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - super.beforeTest(); + resetPlanHistory(); + } + + /** + * @param sqlEngine Sql engine. + */ + protected void setSqlEngine(SqlPlanHistoryTracker.SqlEngine sqlEngine) { + this.sqlEngine = sqlEngine; + } + + /** + * @param isClient Client more flag. + */ + protected void setClientMode(boolean isClient) { + this.isClient = isClient; + } + + /** + * Clears current SQL plan history. + */ + public void resetPlanHistory() { + queryNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); + } + + /** Checks successful JDBC queries. */ + @Test + public void testJdbcQuery() throws SQLException { + if (loc) + return; + + for (int i = 0; i < 2; i++) { + jdbcQuery(SQL); + + checkSqlPlanHistory(getExpectedHistorySize()); + } + } + + /** Checks failed JDBC queries. */ + @Test + public void testJdbcQueryFailed() { + if (loc) + return; + + try { + jdbcQuery(SQL_FAILED); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory(getExpectedHistorySize()); + } + + /** Checks successful SqlFieldsQuery. */ + @Test + public void testSqlFieldsQuery() { + runSuccessfulQuery(sqlFieldsQry); + } + + /** Checks failed SqlFieldsQuery. */ + @Test + public void testSqlFieldsQueryFailed() { + runFailedQuery(sqlFieldsQryFailed); + } + + /** Checks successful cross-cache SqlFieldsQuery. */ + @Test + public void testSqlFieldsCrossCacheQuery() { + runSuccessfulQuery(sqlFieldsQryCrossCache); + } + + /** Checks failed cross-cache SqlFieldsQuery. */ + @Test + public void testSqlFieldsCrossCacheQueryFailed() { + runFailedQuery(sqlFieldsQryCrossCacheFailed); + } + + /** Checks successful SqlFieldsQuery with reduce phase. */ + @Test + public void testSqlFieldsQueryWithReducePhase() { + if (loc) + return; + + cacheQuery(sqlFieldsQryWithReducePhase, "pers"); + + checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); + } + + /** Checks failed SqlFieldsQuery with reduce phase. */ + @Test + public void testSqlFieldsQueryWithReducePhaseFailed() { + if (loc) + return; + + for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { + try { + cacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? i + 1 : 0); + + resetPlanHistory(); + } + } + + /** Checks successful SqlQuery. */ + @Test + public void testSqlQuery() { + runSuccessfulQuery(sqlQry); + } + + /** Checks failed SqlQuery. */ + @Test + public void testSqlQueryFailed() { + runFailedQuery(sqlQryFailed); + } + + /** Checks ScanQuery. */ + @Test + public void testScanQuery() { + runQueryWithoutPlan(scanQry); + } + + /** Checks TextQuery. */ + @Test + public void testTextQuery() { + runQueryWithoutPlan(textQry); + } + + /** Checks DML commands executed via JDBC. */ + @Test + public void testJdbcDml() throws SQLException { + runJdbcDml(dmlCmds); + } + + /** Checks DML commands with multiple operations executed via JDBC. */ + @Test + public void testJdbcDmlMultiOps() throws SQLException { + runJdbcDml(dmlCmdsMultiOps); + } + + /** Checks DML commands with joins executed via JDBC. */ + @Test + public void testJdbcDmlWithJoins() throws SQLException { + runJdbcDml(dmlCmdsWithJoins); + } + + /** Checks failed DML commands executed via JDBC. */ + @Test + public void testJdbcDmlFailed() throws SQLException { + executeJdbcDml((stmt) -> { + for (String cmd : dmlCmdsFailed) + try { + stmt.execute(cmd); + } + catch (Exception ignore) { + //No-Op + } + }); + } + + /** Checks DML commands executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDml() { + runSqlFieldsQueryDml(dmlCmds); + } + + /** Checks DML commands with multiple operations executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlMultiOps() { + runSqlFieldsQueryDml(dmlCmdsMultiOps); + } + + /** Checks DML commands with joins executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlWithJoins() { + runSqlFieldsQueryDml(dmlCmdsWithJoins); + } + + /** Checks failed DML commands executed via SqlFieldsQuery. */ + @Test + public void testSqlFieldsDmlFailed() { + executeSqlFieldsQueryDml(cache -> { + for (String cmd : dmlCmdsFailed) { + try { + cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); + } + catch (Exception ignore) { + //No-Op + } + } + }); + } + + /** Checks that older plan entries are evicted when maximum history size is reached. */ + @Test + public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { + if (loc || (!loc && isFullyFetched)) + return; + + for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { + try { + cacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); + } + catch (Exception ignore) { + //No-Op + } + } + + GridTestUtils.waitForCondition(() -> getSqlPlanHistoryValues().size() == PLAN_HISTORY_SIZE, 1000); + + checkSqlPlanHistory((isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : PLAN_HISTORY_SIZE); + + Set qrys = getSqlPlanHistoryValues().stream().map(SqlPlan::query).collect(Collectors.toSet()); + + for (int i = 1; i <= PLAN_HISTORY_EXCESS; i++) + assertFalse(qrys.contains(SQL + " where A.fail=" + i)); + } + + /** Checks that SQL plan history remains empty if history size is set to zero. */ + @Test + public void testEmptyPlanHistory() { + queryNode().context().query().runningQueryManager().planHistoryTracker().setHistorySize(0); + + executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); + + assertTrue(getSqlPlanHistoryValues().isEmpty()); + } + + /** + * @param qry Query. + */ + public void runSuccessfulQuery(Query qry) { + executeQuery(qry, (q) -> { + for (int i = 0; i < 2; i++) { + cacheQuery(q, "A"); + + checkSqlPlanHistory(getExpectedHistorySize()); + } + }); + } + + /** + * @param qry Query. + */ + public void runFailedQuery(Query qry) { + executeQuery(qry, (q) -> { + try { + cacheQuery(q, "A"); + } + catch (Exception ignore) { + //No-Op + } + + checkSqlPlanHistory(getExpectedHistorySize()); + }); + } + + /** + * @param qry Query. + */ + public void runQueryWithoutPlan(Query qry) { + executeQuery(qry, (q) -> { + cacheQuery(q, "A"); + + checkSqlPlanHistory(0); + }); + } + + /** + * @param qry Query. + * @param task Task to execute. + */ + public void executeQuery(Query qry, Consumer task) { + if (isClient && loc) + return; + + qry.setLocal(loc); + + task.accept(qry); + } + + /** + * @param qry Query. + */ + private void jdbcQuery(String qry) throws SQLException { + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + if (!isFullyFetched) + stmt.setFetchSize(1); + + ResultSet rs = stmt.executeQuery(qry); + + assertTrue(rs.next()); + } + } + + /** + * @param qry Query. + * @param cacheName Cache name. + */ + public void cacheQuery(Query qry, String cacheName) { + IgniteCache cache = queryNode().getOrCreateCache(cacheName); + + if (isFullyFetched) + assertFalse(cache.query(qry).getAll().isEmpty()); + else { + qry.setPageSize(1); + + assertTrue(cache.query(qry).iterator().hasNext()); + + cache.query(qry).iterator().next(); + } + } + + /** + * @param cmds Set of DML commands. + */ + public void runJdbcDml(List cmds) throws SQLException { + executeJdbcDml((stmt) -> { + for (String cmd : cmds) { + try { + stmt.execute(cmd); + } + catch (SQLException e) { + throw new RuntimeException(e); + } + } + }); + } + + /** + * @param task Task to execute. + */ + public void executeJdbcDml(Consumer task) throws SQLException { + if (loc || !isFullyFetched) + return; + + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + task.accept(stmt); + } + + checkSqlPlanHistoryDml(3); + } + + /** + * @param cmds Set of DML commands. + */ + public void runSqlFieldsQueryDml(List cmds) { + executeSqlFieldsQueryDml(cache -> + cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc)))); + } + + /** + * @param task Task to execute. + */ + public void executeSqlFieldsQueryDml(Consumer> task) { + if (isClient && loc || !isFullyFetched) + return; + + IgniteCache cache = queryNode().getOrCreateCache("A"); + + task.accept(cache); + + checkSqlPlanHistoryDml(3); + } + + /** */ + public Collection getSqlPlanHistoryValues() { + return queryNode().context().query().runningQueryManager().planHistoryTracker() + .sqlPlanHistory().values(); + } + + /** */ + public int getExpectedHistorySize() { + return (isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : 1; + } + + /** + * Prepares SQL plan history entries for futher checking. + * + * @param size Number of SQL plan entries expected to be in the history. + */ + public void checkSqlPlanHistory(int size) { + Collection sqlPlans = getSqlPlanHistoryValues(); + + assertNotNull(sqlPlans); + + checkMetrics(size, sqlPlans); + } + + /** + * Prepares SQL plan history entries for futher checking (DML operations). + * + * @param size Number of SQL plan entries expected to be in the history. + */ + public void checkSqlPlanHistoryDml(int size) { + Collection sqlPlans = getSqlPlanHistoryValues(); + + assertNotNull(sqlPlans); + + if (sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) { + Collection sqlPlans0 = new ArrayList<>(); + + for (String str : dmlCheckStrings) + sqlPlans0.addAll(sqlPlans.stream().filter(p -> p.plan().contains(str)).collect(Collectors.toList())); + + sqlPlans = sqlPlans0; + } + + checkMetrics(size, sqlPlans); + } + + /** + * Checks metrics of provided SQL plan history entries. + * + * @param size Number of SQL plan entries expected to be in the history. + * @param sqlPlans Sql plans recorded in the history. + */ + public void checkMetrics(int size, Collection sqlPlans) { + if (size == 1 && sqlPlans.size() == 2) { + String plan1 = new ArrayList<>(sqlPlans).get(0).plan(); + String plan2 = new ArrayList<>(sqlPlans).get(1).plan(); + + assertTrue(plan2.contains(plan1) && plan2.contains("/* scanCount")); + } + else + assertTrue(size == sqlPlans.size()); + + if (size == 0) + return; + + for (SqlPlan plan : sqlPlans) { + assertEquals(loc, plan.local()); + assertEquals(sqlEngine.toString(), plan.engine()); + + assertNotNull(plan.plan()); + assertNotNull(plan.query()); + assertNotNull(plan.schema()); + + assertTrue(plan.startTime() > 0); + } + } + + /** */ + private static class Person { + /** */ + @QuerySqlField(index = true) + int orgId; + + /** */ + @QuerySqlField(index = true) + String name; + + /** + * @param orgId Organization ID. + * @param name Name. + */ + public Person(int orgId, String name) { + this.orgId = orgId; + this.name = name; + } + } + + /** */ + private static class Organization { + /** */ + @QuerySqlField + String name; + + /** + * @param name Organization name. + */ + public Organization(String name) { + this.name = name; + } + } - setSqlEngine(SqlPlanHistoryTracker.SqlEngine.CALCITE); + /** */ + public static class Functions { + /** */ + @QuerySqlFunction + public static int fail() { + throw new IgniteSQLException("SQL function fail for test purpuses"); + } } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java index 19ec8ed069aa0..49d32444a0fce 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java @@ -17,728 +17,21 @@ package org.apache.ignite.internal.processors.query.calcite; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.query.Query; -import org.apache.ignite.cache.query.ScanQuery; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.cache.query.SqlQuery; -import org.apache.ignite.cache.query.TextQuery; -import org.apache.ignite.cache.query.annotations.QuerySqlField; -import org.apache.ignite.cache.query.annotations.QuerySqlFunction; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.SqlConfiguration; import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; -import org.apache.ignite.internal.processors.query.running.SqlPlan; import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; /** Tests for SQL plan history (H2 engine). */ -@RunWith(Parameterized.class) -public class SqlPlanHistoryH2SelfTest extends GridCommonAbstractTest { - /** SQL plan history size. */ - private static final int PLAN_HISTORY_SIZE = 10; - - /** Plan history size excess. */ - private static final int PLAN_HISTORY_EXCESS = 2; - - /** Simple SQL query. */ - private static final String SQL = "SELECT * FROM A.String"; - - /** Failed SQL query. */ - private static final String SQL_FAILED = "select * from A.String where A.fail()=1"; - - /** Cross-cache SQL query. */ - private static final String SQL_CROSS_CACHE = "SELECT * FROM B.String"; - - /** Failed cross-cache SQL query. */ - private static final String SQL_CROSS_CACHE_FAILED = "select * from A.String where A.fail()=1"; - - /** SQL query with reduce phase. */ - private static final String SQL_WITH_REDUCE_PHASE = "select o.name n1, p.name n2 from \"pers\".Person p, " + - "\"org\".Organization o where p.orgId=o._key and o._key=101" + - " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + - " where p.orgId=o._key and o._key=102"; - - /** Set of simple DML commands. */ - private final List dmlCmds = Arrays.asList( - "insert into A.String (_key, _val) values(101, '101')", - "update A.String set _val='111' where _key=101", - "delete from A.String where _key=101" - ); - - /** Set of DML commands with multiple operations. */ - private final List dmlCmdsMultiOps = Arrays.asList( - "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", - "update A.String set _val = case _key " + - "when 101 then '111' " + - "when 102 then '112' " + - "when 103 then '113' " + - "end " + - "where _key in (101, 102, 103)", - "delete from A.String where _key in (101, 102, 103)" - ); - - /** Set of DML commands with joins. */ - private final List dmlCmdsWithJoins = Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name " + - "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", - "update A.String set _val = 'updated' where _key in " + - "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", - "delete from A.String where _key in (select orgId from \"pers\".Person)" - ); - - /** Set of failed DML commands. */ - private final List dmlCmdsFailed = Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + - "where A.fail()=1", - "update A.String set _val = 'failed' where A.fail()=1", - "delete from A.String where A.fail()=1" - ); - - /** Strings for checking SQL plan history after executing DML commands. */ - private final List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); - - /** Successful SqlFieldsQuery. */ - private final SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); - - /** Failed SqlFieldsQuery. */ - private final SqlFieldsQuery sqlFieldsQryFailed = new SqlFieldsQuery(SQL_FAILED); - - /** Successful cross-cache SqlFieldsQuery. */ - private final SqlFieldsQuery sqlFieldsCrossCacheQry = new SqlFieldsQuery(SQL_CROSS_CACHE); - - /** Failed cross-cache SqlFieldsQuery. */ - private final SqlFieldsQuery sqlFieldsCrossCacheQryFailed = new SqlFieldsQuery(SQL_CROSS_CACHE_FAILED); - - /** Successful SqlFieldsQuery with reduce phase. */ - private final SqlFieldsQuery sqlFieldsQryWithReducePhase = new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE) - .setDistributedJoins(true); - - /** Failed SqlFieldsQueries with reduce phase. */ - private final SqlFieldsQuery[] sqlFieldsQryWithReducePhaseFailed = F.asArray( - new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=101", "fail()")).setDistributedJoins(true), - new SqlFieldsQuery(SQL_WITH_REDUCE_PHASE.replace("o._key=102", "fail()")).setDistributedJoins(true) - ); - - /** Successful SqlQuery. */ - private final SqlQuery sqlQry = new SqlQuery<>("String", "from String"); - - /** Failed SqlQuery. */ - private final SqlQuery sqlQryFailed = new SqlQuery<>("String", "from String where fail()=1"); - - /** ScanQuery. */ - private final ScanQuery scanQry = new ScanQuery<>(); - - /** TextQuery. */ - private final TextQuery textQry = new TextQuery<>("String", "2"); - - /** Client mode flag. */ - private boolean isClient; - - /** SQL engine. */ - protected SqlPlanHistoryTracker.SqlEngine sqlEngine = SqlPlanHistoryTracker.SqlEngine.H2; - - /** Local query flag. */ - @Parameterized.Parameter - public boolean loc; - - /** Fully-fetched query flag. */ - @Parameterized.Parameter(1) - public boolean isFullyFetched; - - /** */ - @Parameterized.Parameters(name = "loc={0}, fullyFetched={1}") - public static Collection params() { - return Arrays.asList(new Object[][] { - {true, true}, {true, false}, {false, true}, {false, false} - }); - } - +public class SqlPlanHistoryH2SelfTest extends SqlPlanHistoryCalciteSelfTest { /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - QueryEngineConfigurationEx engCfg = configureSqlEngine(); - - cfg.setSqlConfiguration(new SqlConfiguration() - .setSqlPlanHistorySize(PLAN_HISTORY_SIZE) - .setQueryEnginesConfiguration(engCfg) - ); - - return cfg.setCacheConfiguration( - configureCahce("A", Integer.class, String.class), - configureCahce("B", Integer.class, String.class), - configureCahce("pers", Integer.class, Person.class), - configureCahce("org", Integer.class, Organization.class) - ); - } - - /** */ - protected QueryEngineConfigurationEx configureSqlEngine() { + @Override protected QueryEngineConfigurationEx configureSqlEngine() { return new IndexingQueryEngineConfiguration(); } - /** - * @param name Name. - * @param idxTypes Index types. - * @return Cache configuration. - */ - @SuppressWarnings("unchecked") - private CacheConfiguration configureCahce(String name, Class... idxTypes) { - return new CacheConfiguration() - .setName(name) - .setIndexedTypes(idxTypes) - .setSqlFunctionClasses(Functions.class); - } - - /** - * @return Ignite instance for quering. - */ - protected IgniteEx queryNode() { - IgniteEx node = grid(0); - - assertFalse(node.context().clientNode()); - - return node; - } - - /** - * Starts Ignite instance. - * - * @throws Exception In case of failure. - */ - protected void startTestGrid() throws Exception { - startGrid(0); - } - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - - startTestGrid(); - - IgniteCache cacheA = queryNode().cache("A"); - IgniteCache cacheB = queryNode().cache("B"); - - for (int i = 0; i < 100; i++) { - cacheA.put(i, String.valueOf(i)); - cacheB.put(i, String.valueOf(i)); - } - - IgniteCache cachePers = queryNode().cache("pers"); - IgniteCache cacheOrg = queryNode().cache("org"); - - cacheOrg.put(101, new Organization("o1")); - cacheOrg.put(102, new Organization("o2")); - cachePers.put(103, new Person(101, "p1")); - cachePers.put(104, new Person(102, "p2")); - cachePers.put(105, new Person(103, "p3")); - } - - /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { - super.afterTestsStopped(); - - stopAllGrids(); - } - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - resetPlanHistory(); - } - - /** - * @param sqlEngine Sql engine. - */ - protected void setSqlEngine(SqlPlanHistoryTracker.SqlEngine sqlEngine) { - this.sqlEngine = sqlEngine; - } - - /** - * @param isClient Client more flag. - */ - protected void setClientMode(boolean isClient) { - this.isClient = isClient; - } - - /** */ - public void resetPlanHistory() { - queryNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); - } - - /** Checks successful JDBC queries. */ - @Test - public void testJdbcQuery() throws SQLException { - if (loc) - return; - - for (int i = 0; i < 2; i++) { - jdbcQuery(SQL); - - checkSqlPlanHistory(getExpectedHistorySize()); - } - } - - /** Checks failed JDBC queries. */ - @Test - public void testJdbcQueryFailed() { - if (loc) - return; - - try { - jdbcQuery(SQL_FAILED); - } - catch (Exception ignore) { - //No-Op - } - - checkSqlPlanHistory(getExpectedHistorySize()); - } - - /** Checks successful SqlFieldsQuery. */ - @Test - public void testSqlFieldsQuery() { - runSuccessfulQuery(sqlFieldsQry); - } - - /** Checks failed SqlFieldsQuery. */ - @Test - public void testSqlFieldsQueryFailed() { - runFailedQuery(sqlFieldsQryFailed); - } - - /** Checks successful cross-cache SqlFieldsQuery. */ - @Test - public void testSqlFieldsCrossCacheQuery() { - runSuccessfulQuery(sqlFieldsCrossCacheQry); - } - - /** Checks failed cross-cache SqlFieldsQuery. */ - @Test - public void testSqlFieldsCrossCacheQueryFailed() { - runFailedQuery(sqlFieldsCrossCacheQryFailed); - } - - /** Checks successful SqlFieldsQuery with reduce phase. */ - @Test - public void testSqlFieldsQueryWithReducePhase() { - if (loc) - return; - - cacheQuery(sqlFieldsQryWithReducePhase, "pers"); - - checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); - } - - /** Checks failed SqlFieldsQuery with reduce phase. */ - @Test - public void testSqlFieldsQueryWithReducePhaseFailed() { - if (loc) - return; - - for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { - try { - cacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); - } - catch (Exception ignore) { - //No-Op - } - - checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? i + 1 : 0); - - resetPlanHistory(); - } - } - - /** Checks successful SqlQuery. */ - @Test - public void testSqlQuery() { - runSuccessfulQuery(sqlQry); - } - - /** Checks failed SqlQuery. */ - @Test - public void testSqlQueryFailed() { - runFailedQuery(sqlQryFailed); - } - - /** Checks ScanQuery. */ - @Test - public void testScanQuery() { - runQueryWithoutPlan(scanQry); - } - - /** Checks TextQuery. */ - @Test - public void testTextQuery() { - runQueryWithoutPlan(textQry); - } - - /** Checks DML commands executed via JDBC. */ - @Test - public void testJdbcDml() throws SQLException { - runJdbcDml(dmlCmds); - } - - /** Checks DML commands with multiple operations executed via JDBC. */ - @Test - public void testJdbcDmlMultiOps() throws SQLException { - runJdbcDml(dmlCmdsMultiOps); - } - - /** Checks DML commands with joins executed via JDBC. */ - @Test - public void testJdbcDmlWithJoins() throws SQLException { - runJdbcDml(dmlCmdsWithJoins); - } - - /** Checks failed DML commands executed via JDBC. */ - @Test - public void testJdbcDmlFailed() throws SQLException { - executeJdbcDml((stmt) -> { - for (String cmd : dmlCmdsFailed) - try { - stmt.execute(cmd); - } - catch (Exception ignore) { - //No-Op - } - }); - } - - /** Checks DML commands executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDml() { - runSqlFieldsQueryDml(dmlCmds); - } - - /** Checks DML commands with multiple operations executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDmlMultiOps() { - runSqlFieldsQueryDml(dmlCmdsMultiOps); - } - - /** Checks DML commands with joins executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDmlWithJoins() { - runSqlFieldsQueryDml(dmlCmdsWithJoins); - } - - /** Checks failed DML commands executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDmlFailed() { - executeSqlFieldsQueryDml(cache -> { - for (String cmd : dmlCmdsFailed) { - try { - cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); - } - catch (Exception ignore) { - //No-Op - } - } - }); - } - - /** Checks that older plan entries are evicted when maximum history size is reached. */ - @Test - public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { - if (loc || (!loc && isFullyFetched)) - return; - - for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { - try { - cacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); - } - catch (Exception ignore) { - //No-Op - } - } - - GridTestUtils.waitForCondition(() -> getSqlPlanHistoryValues().size() == PLAN_HISTORY_SIZE, 1000); - - checkSqlPlanHistory((isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : PLAN_HISTORY_SIZE); - - Set qrys = getSqlPlanHistoryValues().stream().map(SqlPlan::query).collect(Collectors.toSet()); - - for (int i = 1; i <= PLAN_HISTORY_EXCESS; i++) - assertFalse(qrys.contains(SQL + " where A.fail=" + i)); - } - - /** - * @param qry Query. - */ - public void runSuccessfulQuery(Query qry) { - executeQuery(qry, () -> { - for (int i = 0; i < 2; i++) { - cacheQuery(qry, "A"); - - checkSqlPlanHistory(getExpectedHistorySize()); - } - }); - } - - /** - * @param qry Query. - */ - public void runFailedQuery(Query qry) { - executeQuery(qry, () -> { - try { - cacheQuery(qry, "A"); - } - catch (Exception ignore) { - //No-Op - } - - checkSqlPlanHistory(getExpectedHistorySize()); - }); - } - - /** - * @param qry Query. - */ - public void runQueryWithoutPlan(Query qry) { - executeQuery(qry, () -> { - cacheQuery(qry, "A"); - - checkSqlPlanHistory(0); - }); - } - - /** - * @param qry Query. - * @param task Task to execute. - */ - public void executeQuery(Query qry, Runnable task) { - if (isClient && loc) - return; - - qry.setLocal(loc); - - task.run(); - } - - /** - * @param qry Query. - */ - private void jdbcQuery(String qry) throws SQLException { - try ( - Connection conn = GridTestUtils.connect(queryNode(), null); - Statement stmt = conn.createStatement() - ) { - if (!isFullyFetched) - stmt.setFetchSize(1); - - ResultSet rs = stmt.executeQuery(qry); - - assertTrue(rs.next()); - } - } - - /** - * @param qry Query. - * @param cacheName Cache name. - */ - public void cacheQuery(Query qry, String cacheName) { - IgniteCache cache = queryNode().getOrCreateCache(cacheName); - - if (isFullyFetched) - assertFalse(cache.query(qry).getAll().isEmpty()); - else { - qry.setPageSize(1); - - assertTrue(cache.query(qry).iterator().hasNext()); - - cache.query(qry).iterator().next(); - } - } - - /** - * @param cmds Set of DML commands. - */ - public void runJdbcDml(List cmds) throws SQLException { - executeJdbcDml((stmt) -> { - for (String cmd : cmds) { - try { - stmt.execute(cmd); - } - catch (SQLException e) { - throw new RuntimeException(e); - } - } - }); - } - - /** - * @param task Task to execute. - */ - public void executeJdbcDml(Consumer task) throws SQLException { - if (loc || !isFullyFetched) - return; - - try ( - Connection conn = GridTestUtils.connect(queryNode(), null); - Statement stmt = conn.createStatement() - ) { - task.accept(stmt); - } - - checkSqlPlanHistoryDml(3); - } - - /** - * @param cmds Set of DML commands. - */ - public void runSqlFieldsQueryDml(List cmds) { - executeSqlFieldsQueryDml(cache -> - cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc)))); - } - - /** - * @param task Task to execute. - */ - public void executeSqlFieldsQueryDml(Consumer> task) { - if (isClient && loc || !isFullyFetched) - return; - - IgniteCache cache = queryNode().getOrCreateCache("A"); - - task.accept(cache); - - checkSqlPlanHistoryDml(3); - } - - /** */ - public Collection getSqlPlanHistoryValues() { - return queryNode().context().query().runningQueryManager().planHistoryTracker() - .sqlPlanHistory().values(); - } - - /** */ - public int getExpectedHistorySize() { - return (isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : 1; - } - - /** - * @param size Number of SQL plan entries expected to be in the history. - */ - public void checkSqlPlanHistory(int size) { - Collection sqlPlans = getSqlPlanHistoryValues(); - - assertNotNull(sqlPlans); - - checkMetrics(size, sqlPlans); - } - - /** - * @param size Number of SQL plan entries expected to be in the history. - */ - public void checkSqlPlanHistoryDml(int size) { - Collection sqlPlans = getSqlPlanHistoryValues(); - - assertNotNull(sqlPlans); - - if (sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) { - Collection sqlPlans0 = new ArrayList<>(); - - for (String str : dmlCheckStrings) - sqlPlans0.addAll(sqlPlans.stream().filter(p -> p.plan().contains(str)).collect(Collectors.toList())); - - sqlPlans = sqlPlans0; - } - - checkMetrics(size, sqlPlans); - } - - /** - * @param size Number of SQL plan entries expected to be in the history. - * @param sqlPlans Sql plans recorded in the history. - */ - public void checkMetrics(int size, Collection sqlPlans) { - if (size == 1 && sqlPlans.size() == 2) { - String plan1 = new ArrayList<>(sqlPlans).get(0).plan(); - String plan2 = new ArrayList<>(sqlPlans).get(1).plan(); - - assertTrue(plan2.contains(plan1) && plan2.contains("/* scanCount")); - } - else - assertTrue(size == sqlPlans.size()); - - if (size == 0) - return; - - for (SqlPlan plan : sqlPlans) { - assertEquals(loc, plan.local()); - assertEquals(sqlEngine.toString(), plan.engine()); - - assertNotNull(plan.plan()); - assertNotNull(plan.query()); - assertNotNull(plan.schema()); - - assertTrue(plan.startTime() > 0); - } - } - - /** */ - private static class Person { - /** */ - @QuerySqlField(index = true) - int orgId; - - /** */ - @QuerySqlField(index = true) - String name; - - /** - * @param orgId Organization ID. - * @param name Name. - */ - public Person(int orgId, String name) { - this.orgId = orgId; - this.name = name; - } - } - - /** */ - private static class Organization { - /** */ - @QuerySqlField - String name; - - /** - * @param name Organization name. - */ - public Organization(String name) { - this.name = name; - } - } + super.beforeTest(); - /** */ - public static class Functions { - /** */ - @QuerySqlFunction - public static int fail() { - throw new IgniteSQLException("SQL function fail for test purpuses"); - } + setSqlEngine(SqlPlanHistoryTracker.SqlEngine.H2); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java index fa264f383a949..757f233f42854 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java @@ -57,10 +57,10 @@ NumericTypesPrecisionsTest.class, SqlPlanHistoryConfigTest.class, - SqlPlanHistoryH2SelfTest.class, - SqlPlanHistoryH2FromClientSelfTest.class, SqlPlanHistoryCalciteSelfTest.class, SqlPlanHistoryCalciteFromClientSelfTest.class, + SqlPlanHistoryH2SelfTest.class, + SqlPlanHistoryH2FromClientSelfTest.class, }) public class IgniteCalciteTestSuite { } diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java index f8afd45bb40bc..97960fb221b27 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/SqlConfiguration.java @@ -128,7 +128,7 @@ public int getSqlPlanHistorySize() { * Sets number of SQL plan history elements kept in memory. If not explicitly set, then default value is {@link * #DFLT_SQL_PLAN_HISTORY_SIZE}. * - * @param size Number of SQL query history elements kept in memory. + * @param size Number of SQL plan history elements kept in memory. * @return {@code this} for chaining. */ public SqlConfiguration setSqlPlanHistorySize(int size) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java index 699af8c9dd4a3..fc2fe13e6c43b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java @@ -195,12 +195,12 @@ public RunningQueryManager(GridKernalContext ctx) { histSz = ctx.config().getSqlConfiguration().getSqlQueryHistorySize(); closure = ctx.closure(); - planHistSz = ctx.config().getSqlConfiguration().getSqlPlanHistorySize(); - qryHistTracker = new QueryHistoryTracker(histSz); heavyQrysTracker = ctx.query().moduleEnabled() ? new HeavyQueriesTracker(ctx) : null; + planHistSz = ctx.config().getSqlConfiguration().getSqlPlanHistorySize(); + planHistTracker = new SqlPlanHistoryTracker(planHistSz); ctx.systemView().registerView(SQL_QRY_VIEW, SQL_QRY_VIEW_DESC, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java index 87c397f4a0779..f5082f130e4a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java @@ -17,7 +17,7 @@ package org.apache.ignite.internal.processors.query.running; -/** Representation of an SQL plan history entry. */ +/** Representation of an entry in SQL plan history. */ public class SqlPlan { /** */ private final SqlPlanKey key; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index 6e0478aea58f9..ff4a304915531 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -27,7 +27,7 @@ public class SqlPlanHistoryTracker { private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; /** SQL plan history size. */ - private final int historySize; + private int historySize; /** * @param historySize SQL plan history size. @@ -63,6 +63,13 @@ public Map sqlPlanHistory() { return Collections.unmodifiableMap(sqlPlanHistory); } + /** + * @param historySize History size. + */ + public void setHistorySize(int historySize) { + this.historySize = historySize; + } + /** */ public enum SqlEngine { /** */ From ed7380c65078eb1778cba9250b69a3766a750d20 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Tue, 6 Aug 2024 16:50:07 +1000 Subject: [PATCH 14/33] IGNITE-22557 Adding plans for DML commands moved from IgniteH2Indexing#executeUpdate0 to IgniteH2Indexing#executeUpdate --- .../processors/query/h2/IgniteH2Indexing.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 6851f158dcf98..dfa92f8ec5dbf 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -2096,6 +2096,17 @@ private UpdateResult executeUpdate( IndexingQueryFilter filters, GridQueryCancel cancel ) throws IgniteCheckedException { + SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); + + planHistTracker.addPlan( + dml.plan().plan(), + qryDesc.sql(), + qryDesc.schemaName(), + loc, + U.currentTimeMillis(), + SqlPlanHistoryTracker.SqlEngine.H2 + ); + Object[] errKeys = null; long items = 0; @@ -2167,17 +2178,6 @@ private UpdateResult executeUpdate0( ) throws IgniteCheckedException { UpdatePlan plan = dml.plan(); - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( - plan.plan(), - qryDesc.sql(), - qryDesc.schemaName(), - loc, - U.currentTimeMillis(), - SqlPlanHistoryTracker.SqlEngine.H2 - ); - UpdateResult fastUpdateRes = plan.processFast(qryParams.arguments()); if (fastUpdateRes != null) From 9cb98f9e793546be0a0f994c54802ef92d40aa1f Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Tue, 10 Sep 2024 22:21:09 +1000 Subject: [PATCH 15/33] IGNITE-22557 Changes made according to code review --- .../processors/query/calcite/RootQuery.java | 5 - .../calcite/exec/ExecutionServiceImpl.java | 8 +- .../SqlPlanHistoryClientIntegrationTest.java} | 10 +- .../SqlPlanHistoryConfigIntegrationTest.java} | 4 +- ...qlPlanHistoryH2ClientIntegrationTest.java} | 10 +- .../SqlPlanHistoryH2IntegrationTest.java} | 7 +- .../SqlPlanHistoryIntegrationTest.java} | 159 +++++-------- .../testsuites/IntegrationTestSuite.java | 10 + .../query/running/RunningQueryManager.java | 2 +- .../processors/query/running/SqlPlan.java | 84 +++++-- .../query/running/SqlPlanHistoryTracker.java | 20 +- .../processors/query/running/SqlPlanKey.java | 95 -------- .../query/running/SqlPlanValue.java | 46 ---- .../processors/query/h2/IgniteH2Indexing.java | 224 ++++++++++-------- .../processors/query/h2/dml/DmlArguments.java | 7 +- .../processors/query/h2/dml/FastUpdate.java | 15 -- .../processors/query/h2/dml/UpdatePlan.java | 80 ------- .../h2/twostep/GridMapQueryExecutor.java | 9 +- .../h2/twostep/GridReduceQueryExecutor.java | 9 +- 19 files changed, 276 insertions(+), 528 deletions(-) rename modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/{SqlPlanHistoryCalciteFromClientSelfTest.java => integration/SqlPlanHistoryClientIntegrationTest.java} (85%) rename modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/{SqlPlanHistoryConfigTest.java => integration/SqlPlanHistoryConfigIntegrationTest.java} (91%) rename modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/{SqlPlanHistoryH2FromClientSelfTest.java => integration/SqlPlanHistoryH2ClientIntegrationTest.java} (84%) rename modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/{SqlPlanHistoryH2SelfTest.java => integration/SqlPlanHistoryH2IntegrationTest.java} (82%) rename modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/{SqlPlanHistoryCalciteSelfTest.java => integration/SqlPlanHistoryIntegrationTest.java} (80%) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java index 951c9490a5fdb..242e355d22c7e 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/RootQuery.java @@ -456,11 +456,6 @@ public long remainingTime() { return curTimeout <= 0 ? 0 : curTimeout; } - /** */ - public long startTime() { - return startTs; - } - /** */ @Override public String toString() { return S.toString(RootQuery.class, this); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java index 654ad847033cb..8444de4dcf075 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionServiceImpl.java @@ -110,7 +110,6 @@ import org.apache.ignite.internal.processors.query.calcite.util.ConvertingClosableIterator; import org.apache.ignite.internal.processors.query.calcite.util.ListFieldsQueryCursor; import org.apache.ignite.internal.processors.query.running.HeavyQueriesTracker; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.security.SecurityUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; @@ -686,15 +685,12 @@ private ListFieldsQueryCursor mapAndExecutePlan( ); } - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( + ctx.query().runningQueryManager().planHistoryTracker().addPlan( plan.textPlan(), qry.sql(), qry.context().schemaName(), qry.context().isLocal(), - qry.startTime(), - SqlPlanHistoryTracker.SqlEngine.CALCITE + CalciteQueryEngineConfiguration.ENGINE_NAME ); QueryProperties qryProps = qry.context().unwrap(QueryProperties.class); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java similarity index 85% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java rename to modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java index b51bd7c66b231..7734a068b29d0 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteFromClientSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite; +package org.apache.ignite.internal.processors.query.calcite.integration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.testframework.junits.JUnitAssertAware; /** Tests for SQL plan history from client (Calcite engine). */ -public class SqlPlanHistoryCalciteFromClientSelfTest extends SqlPlanHistoryCalciteSelfTest { +public class SqlPlanHistoryClientIntegrationTest extends SqlPlanHistoryIntegrationTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -31,7 +31,7 @@ public class SqlPlanHistoryCalciteFromClientSelfTest extends SqlPlanHistoryCalci /** {@inheritDoc} */ @Override protected IgniteEx queryNode() { - IgniteEx node = grid(1); + IgniteEx node = grid(2); JUnitAssertAware.assertTrue(node.context().clientNode()); @@ -40,7 +40,7 @@ public class SqlPlanHistoryCalciteFromClientSelfTest extends SqlPlanHistoryCalci /** {@inheritDoc} */ @Override protected void startTestGrid() throws Exception { - startGrid(0); - startClientGrid(1); + startGrids(2); + startClientGrid(2); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java similarity index 91% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java rename to modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java index 10f6f347ca3fe..28213ed444b59 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryConfigTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite; +package org.apache.ignite.internal.processors.query.calcite.integration; import org.apache.ignite.Ignite; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; /** Test for Sql plan history configuration. */ -public class SqlPlanHistoryConfigTest extends GridCommonAbstractTest { +public class SqlPlanHistoryConfigIntegrationTest extends GridCommonAbstractTest { /** Sql plan history size in the XML Spring config. */ private static final int SQL_PLAN_HISTORY_SIZE_XML_CONFIG = 10; diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java similarity index 84% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java rename to modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java index 9a79114f94e29..a951c995c00d9 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2FromClientSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite; +package org.apache.ignite.internal.processors.query.calcite.integration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.testframework.junits.JUnitAssertAware; /** Tests for SQL plan history from client (H2 engine). */ -public class SqlPlanHistoryH2FromClientSelfTest extends SqlPlanHistoryH2SelfTest { +public class SqlPlanHistoryH2ClientIntegrationTest extends SqlPlanHistoryH2IntegrationTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -31,7 +31,7 @@ public class SqlPlanHistoryH2FromClientSelfTest extends SqlPlanHistoryH2SelfTest /** {@inheritDoc} */ @Override protected IgniteEx queryNode() { - IgniteEx node = grid(1); + IgniteEx node = grid(2); JUnitAssertAware.assertTrue(node.context().clientNode()); @@ -40,7 +40,7 @@ public class SqlPlanHistoryH2FromClientSelfTest extends SqlPlanHistoryH2SelfTest /** {@inheritDoc} */ @Override protected void startTestGrid() throws Exception { - startGrid(0); - startClientGrid(1); + startGrids(2); + startClientGrid(2); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java similarity index 82% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java rename to modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java index 49d32444a0fce..3835e24738467 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryH2SelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java @@ -15,14 +15,13 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite; +package org.apache.ignite.internal.processors.query.calcite.integration; import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; /** Tests for SQL plan history (H2 engine). */ -public class SqlPlanHistoryH2SelfTest extends SqlPlanHistoryCalciteSelfTest { +public class SqlPlanHistoryH2IntegrationTest extends SqlPlanHistoryIntegrationTest { /** {@inheritDoc} */ @Override protected QueryEngineConfigurationEx configureSqlEngine() { return new IndexingQueryEngineConfiguration(); @@ -32,6 +31,6 @@ public class SqlPlanHistoryH2SelfTest extends SqlPlanHistoryCalciteSelfTest { @Override protected void beforeTest() throws Exception { super.beforeTest(); - setSqlEngine(SqlPlanHistoryTracker.SqlEngine.H2); + setSqlEngine(IndexingQueryEngineConfiguration.ENGINE_NAME); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java similarity index 80% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java rename to modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 281288c52242e..f00ffcc5169cf 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/SqlPlanHistoryCalciteSelfTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite; +package org.apache.ignite.internal.processors.query.calcite.integration; import java.sql.Connection; import java.sql.ResultSet; @@ -40,12 +40,12 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; import org.apache.ignite.internal.processors.query.running.SqlPlan; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -55,7 +55,7 @@ /** Tests for SQL plan history (Calcite engine). */ @RunWith(Parameterized.class) -public class SqlPlanHistoryCalciteSelfTest extends GridCommonAbstractTest { +public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { /** SQL plan history size. */ private static final int PLAN_HISTORY_SIZE = 10; @@ -87,18 +87,6 @@ public class SqlPlanHistoryCalciteSelfTest extends GridCommonAbstractTest { "delete from A.String where _key=101" ); - /** Set of DML commands with multiple operations. */ - private final List dmlCmdsMultiOps = Arrays.asList( - "insert into A.String (_key, _val) values(101, '101'), (102, '102'), (103, '103')", - "update A.String set _val = case _key " + - "when 101 then '111' " + - "when 102 then '112' " + - "when 103 then '113' " + - "end " + - "where _key in (101, 102, 103)", - "delete from A.String where _key in (101, 102, 103)" - ); - /** Set of DML commands with joins. */ private final List dmlCmdsWithJoins = Arrays.asList( "insert into A.String (_key, _val) select o._key, p.name " + @@ -108,17 +96,6 @@ public class SqlPlanHistoryCalciteSelfTest extends GridCommonAbstractTest { "delete from A.String where _key in (select orgId from \"pers\".Person)" ); - /** Set of failed DML commands. */ - private final List dmlCmdsFailed = Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name from \"pers\".Person p, \"org\".Organization o " + - "where A.fail()=1", - "update A.String set _val = 'failed' where A.fail()=1", - "delete from A.String where A.fail()=1" - ); - - /** Strings for checking SQL plan history after executing DML commands. */ - private final List dmlCheckStrings = Arrays.asList("mode=INSERT", "mode=UPDATE", "mode=DELETE"); - /** Successful SqlFieldsQuery. */ private final SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(SQL); @@ -157,7 +134,7 @@ public class SqlPlanHistoryCalciteSelfTest extends GridCommonAbstractTest { private boolean isClient; /** SQL engine. */ - protected SqlPlanHistoryTracker.SqlEngine sqlEngine = SqlPlanHistoryTracker.SqlEngine.CALCITE; + protected String sqlEngine = CalciteQueryEngineConfiguration.ENGINE_NAME; /** Local query flag. */ @Parameterized.Parameter @@ -223,13 +200,24 @@ protected IgniteEx queryNode() { return node; } + /** + * @return Ignite map node. + */ + protected IgniteEx mapNode() { + IgniteEx node = grid(1); + + assertFalse(node.context().clientNode()); + + return node; + } + /** * Starts Ignite instance. * * @throws Exception In case of failure. */ protected void startTestGrid() throws Exception { - startGrid(0); + startGrids(2); } /** {@inheritDoc} */ @@ -253,7 +241,6 @@ protected void startTestGrid() throws Exception { cacheOrg.put(102, new Organization("o2")); cachePers.put(103, new Person(101, "p1")); cachePers.put(104, new Person(102, "p2")); - cachePers.put(105, new Person(103, "p3")); } /** {@inheritDoc} */ @@ -271,7 +258,7 @@ protected void startTestGrid() throws Exception { /** * @param sqlEngine Sql engine. */ - protected void setSqlEngine(SqlPlanHistoryTracker.SqlEngine sqlEngine) { + protected void setSqlEngine(String sqlEngine) { this.sqlEngine = sqlEngine; } @@ -287,6 +274,8 @@ protected void setClientMode(boolean isClient) { */ public void resetPlanHistory() { queryNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); + + mapNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); } /** Checks successful JDBC queries. */ @@ -350,7 +339,7 @@ public void testSqlFieldsQueryWithReducePhase() { cacheQuery(sqlFieldsQryWithReducePhase, "pers"); - checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 3 : 1); + checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 3 : 1); } /** Checks failed SqlFieldsQuery with reduce phase. */ @@ -367,7 +356,7 @@ public void testSqlFieldsQueryWithReducePhaseFailed() { //No-Op } - checkSqlPlanHistory((!isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? i + 1 : 0); + checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? i + 1 : 0); resetPlanHistory(); } @@ -400,66 +389,25 @@ public void testTextQuery() { /** Checks DML commands executed via JDBC. */ @Test public void testJdbcDml() throws SQLException { - runJdbcDml(dmlCmds); - } - - /** Checks DML commands with multiple operations executed via JDBC. */ - @Test - public void testJdbcDmlMultiOps() throws SQLException { - runJdbcDml(dmlCmdsMultiOps); + runJdbcDml(dmlCmds, true); } /** Checks DML commands with joins executed via JDBC. */ @Test public void testJdbcDmlWithJoins() throws SQLException { - runJdbcDml(dmlCmdsWithJoins); - } - - /** Checks failed DML commands executed via JDBC. */ - @Test - public void testJdbcDmlFailed() throws SQLException { - executeJdbcDml((stmt) -> { - for (String cmd : dmlCmdsFailed) - try { - stmt.execute(cmd); - } - catch (Exception ignore) { - //No-Op - } - }); + runJdbcDml(dmlCmdsWithJoins, false); } /** Checks DML commands executed via SqlFieldsQuery. */ @Test public void testSqlFieldsDml() { - runSqlFieldsQueryDml(dmlCmds); - } - - /** Checks DML commands with multiple operations executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDmlMultiOps() { - runSqlFieldsQueryDml(dmlCmdsMultiOps); + runSqlFieldsQueryDml(dmlCmds, true); } /** Checks DML commands with joins executed via SqlFieldsQuery. */ @Test public void testSqlFieldsDmlWithJoins() { - runSqlFieldsQueryDml(dmlCmdsWithJoins); - } - - /** Checks failed DML commands executed via SqlFieldsQuery. */ - @Test - public void testSqlFieldsDmlFailed() { - executeSqlFieldsQueryDml(cache -> { - for (String cmd : dmlCmdsFailed) { - try { - cache.query(new SqlFieldsQuery(cmd).setLocal(loc)); - } - catch (Exception ignore) { - //No-Op - } - } - }); + runSqlFieldsQueryDml(dmlCmdsWithJoins, false); } /** Checks that older plan entries are evicted when maximum history size is reached. */ @@ -477,11 +425,12 @@ public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { } } - GridTestUtils.waitForCondition(() -> getSqlPlanHistoryValues().size() == PLAN_HISTORY_SIZE, 1000); + GridTestUtils.waitForCondition(() -> getSqlPlanHistory(queryNode()).size() == PLAN_HISTORY_SIZE, 1000); - checkSqlPlanHistory((isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : PLAN_HISTORY_SIZE); + checkSqlPlanHistory((isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 0 : + PLAN_HISTORY_SIZE); - Set qrys = getSqlPlanHistoryValues().stream().map(SqlPlan::query).collect(Collectors.toSet()); + Set qrys = getSqlPlanHistory(queryNode()).stream().map(SqlPlan::query).collect(Collectors.toSet()); for (int i = 1; i <= PLAN_HISTORY_EXCESS; i++) assertFalse(qrys.contains(SQL + " where A.fail=" + i)); @@ -494,7 +443,7 @@ public void testEmptyPlanHistory() { executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); - assertTrue(getSqlPlanHistoryValues().isEmpty()); + assertTrue(getSqlPlanHistory(queryNode()).isEmpty()); } /** @@ -588,7 +537,7 @@ public void cacheQuery(Query qry, String cacheName) { /** * @param cmds Set of DML commands. */ - public void runJdbcDml(List cmds) throws SQLException { + public void runJdbcDml(List cmds, boolean isSimpleQry) throws SQLException { executeJdbcDml((stmt) -> { for (String cmd : cmds) { try { @@ -598,13 +547,13 @@ public void runJdbcDml(List cmds) throws SQLException { throw new RuntimeException(e); } } - }); + }, isSimpleQry); } /** * @param task Task to execute. */ - public void executeJdbcDml(Consumer task) throws SQLException { + public void executeJdbcDml(Consumer task, boolean isSimpleQry) throws SQLException { if (loc || !isFullyFetched) return; @@ -615,21 +564,22 @@ public void executeJdbcDml(Consumer task) throws SQLException { task.accept(stmt); } - checkSqlPlanHistoryDml(3); + checkSqlPlanHistoryDml(3, isSimpleQry); } /** * @param cmds Set of DML commands. */ - public void runSqlFieldsQueryDml(List cmds) { - executeSqlFieldsQueryDml(cache -> - cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc)))); + public void runSqlFieldsQueryDml(List cmds, boolean isSimpleQry) { + executeSqlFieldsQueryDml( + cache -> cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc))), + isSimpleQry); } /** * @param task Task to execute. */ - public void executeSqlFieldsQueryDml(Consumer> task) { + public void executeSqlFieldsQueryDml(Consumer> task, boolean isSimpleQry) { if (isClient && loc || !isFullyFetched) return; @@ -637,18 +587,18 @@ public void executeSqlFieldsQueryDml(Consumer> task task.accept(cache); - checkSqlPlanHistoryDml(3); + checkSqlPlanHistoryDml(3, isSimpleQry); } /** */ - public Collection getSqlPlanHistoryValues() { - return queryNode().context().query().runningQueryManager().planHistoryTracker() - .sqlPlanHistory().values(); + public Collection getSqlPlanHistory(IgniteEx node) { + return node.context().query().runningQueryManager().planHistoryTracker() + .sqlPlanHistory().keySet(); } /** */ public int getExpectedHistorySize() { - return (isClient && sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) ? 0 : 1; + return (isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 0 : 1; } /** @@ -657,7 +607,7 @@ public int getExpectedHistorySize() { * @param size Number of SQL plan entries expected to be in the history. */ public void checkSqlPlanHistory(int size) { - Collection sqlPlans = getSqlPlanHistoryValues(); + Collection sqlPlans = getSqlPlanHistory(queryNode()); assertNotNull(sqlPlans); @@ -668,19 +618,24 @@ public void checkSqlPlanHistory(int size) { * Prepares SQL plan history entries for futher checking (DML operations). * * @param size Number of SQL plan entries expected to be in the history. + * @param isSimpleQry Simple query flag. */ - public void checkSqlPlanHistoryDml(int size) { - Collection sqlPlans = getSqlPlanHistoryValues(); + public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { + Collection sqlPlans = getSqlPlanHistory(queryNode()); assertNotNull(sqlPlans); - if (sqlEngine == SqlPlanHistoryTracker.SqlEngine.H2) { - Collection sqlPlans0 = new ArrayList<>(); + if (sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { + String check; + + if (isSimpleQry) + check = "no SELECT queries have been executed."; + else + check = "the following " + (loc ? "local " : "") + "query has been executed:"; - for (String str : dmlCheckStrings) - sqlPlans0.addAll(sqlPlans.stream().filter(p -> p.plan().contains(str)).collect(Collectors.toList())); + sqlPlans = sqlPlans.stream().filter(p -> p.plan().contains(check)).collect(Collectors.toList()); - sqlPlans = sqlPlans0; + checkMetrics((!isSimpleQry && !loc) ? size : 0, getSqlPlanHistory(mapNode())); } checkMetrics(size, sqlPlans); diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index 15c302e337ee1..8c62894dfd0fe 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -59,6 +59,11 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SetOpIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SortAggregateIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlDiagnosticIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryClientIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryConfigIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2ClientIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2IntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.StatisticsCommandDdlIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.StdSqlOperatorsTest; import org.apache.ignite.internal.processors.query.calcite.integration.SystemViewsIntegrationTest; @@ -139,6 +144,11 @@ DdlTransactionCalciteSelfTest.class, MultiLineQueryTest.class, ViewsIntegrationTest.class, + SqlPlanHistoryConfigIntegrationTest.class, + SqlPlanHistoryIntegrationTest.class, + SqlPlanHistoryClientIntegrationTest.class, + SqlPlanHistoryH2IntegrationTest.class, + SqlPlanHistoryH2ClientIntegrationTest.class, }) public class IntegrationTestSuite { } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java index fc2fe13e6c43b..9ddb16b63c5dc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java @@ -215,7 +215,7 @@ public RunningQueryManager(GridKernalContext ctx) { ctx.systemView().registerView(SQL_PLAN_HIST_VIEW, SQL_PLAN_HIST_VIEW_DESC, new SqlPlanHistoryViewWalker(), - planHistTracker.sqlPlanHistory().values(), + planHistTracker.sqlPlanHistory().keySet(), SqlPlanHistoryView::new); MetricRegistryImpl userMetrics = ctx.metric().registry(SQL_USER_QUERIES_REG_NAME); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java index f5082f130e4a0..bac6fdd129c23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java @@ -17,71 +17,103 @@ package org.apache.ignite.internal.processors.query.running; +import java.util.Objects; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; + /** Representation of an entry in SQL plan history. */ public class SqlPlan { - /** */ - private final SqlPlanKey key; + /** SQL plan. */ + private final String plan; - /** */ - private final SqlPlanValue val; + /** Query. */ + private final String qry; + + /** Schema name. */ + private final String schema; + + /** Local query flag. */ + private final boolean loc; + + /** SQL engine. */ + private final String engine; + + /** Query start timestamp. */ + private final long startTime; + + /** Pre-calculated hash code. */ + private final int hash; /** * @param plan SQL plan. * @param qry Query. * @param schema Schema name. * @param loc Local query flag. - * @param startTime Start query timestamp. */ public SqlPlan( String plan, String qry, String schema, boolean loc, - long startTime, - SqlPlanHistoryTracker.SqlEngine engine + String engine ) { - key = new SqlPlanKey(plan, qry, schema, loc); - - val = new SqlPlanValue(startTime, engine); - } + this.plan = plan; + this.qry = qry; + this.schema = schema; + this.loc = loc; + this.engine = engine; - /** */ - public SqlPlanKey key() { - return key; - } + startTime = U.currentTimeMillis(); - /** */ - public SqlPlanValue value() { - return val; + hash = Objects.hash(plan, qry, schema, loc, engine); } /** */ public String plan() { - return key.plan(); + return plan; } /** */ public String query() { - return key.query(); + return qry; } /** */ public String schema() { - return key.schema(); + return schema; } /** */ public boolean local() { - return key.local(); + return loc; } /** */ - public long startTime() { - return val.startTime(); + public String engine() { + return engine; } /** */ - public String engine() { - return val.engine(); + public long startTime() { + return startTime; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return hash; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + SqlPlan plan0 = (SqlPlan)o; + + return F.eq(plan, plan0.plan) && F.eq(qry, plan0.qry) && F.eq(schema, plan0.schema) && F.eq(loc, plan0.loc) + && F.eq(engine, plan0.engine); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index ff4a304915531..fdbc237b915a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -24,7 +24,7 @@ /** Class that manages recording and storing SQL plans. */ public class SqlPlanHistoryTracker { /** SQL plan history. */ - private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; + private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; /** SQL plan history size. */ private int historySize; @@ -43,20 +43,19 @@ public SqlPlanHistoryTracker(int historySize) { * @param qry Query. * @param schema Schema name. * @param loc Local query flag. - * @param startTs Start query timestamp. * @param engine SQL engine. */ - public void addPlan(String plan, String qry, String schema, boolean loc, long startTs, SqlEngine engine) { + public void addPlan(String plan, String qry, String schema, boolean loc, String engine) { if (historySize <= 0) return; - SqlPlan sqlPlan = new SqlPlan(plan, qry, schema, loc, startTs, engine); + SqlPlan sqlPlan = new SqlPlan(plan, qry, schema, loc, engine); - sqlPlanHistory.put(sqlPlan.key(), sqlPlan); + sqlPlanHistory.put(sqlPlan, sqlPlan.startTime()); } /** */ - public Map sqlPlanHistory() { + public Map sqlPlanHistory() { if (historySize <= 0) return Collections.emptyMap(); @@ -69,13 +68,4 @@ public Map sqlPlanHistory() { public void setHistorySize(int historySize) { this.historySize = historySize; } - - /** */ - public enum SqlEngine { - /** */ - CALCITE, - - /** */ - H2 - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java deleted file mode 100644 index 24c044482ae52..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanKey.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.running; - -import org.apache.ignite.internal.util.typedef.F; - -/** SQL plan history entry key. */ -public class SqlPlanKey { - /** Plan. */ - private final String plan; - - /** Query. */ - private final String qry; - - /** Schema name. */ - private final String schema; - - /** Local query flag. */ - private final boolean loc; - - /** Pre-calculated hash code. */ - private final int hash; - - /** - * @param plan SQL Plan. - * @param qry Query. - * @param schema Schema name. - * @param loc Local query flag. - */ - public SqlPlanKey(String plan, String qry, String schema, boolean loc) { - assert plan != null; - assert qry != null; - assert schema != null; - - this.plan = plan; - this.qry = qry; - this.schema = schema; - this.loc = loc; - - hash = 31 * (31 * plan.hashCode() + qry.hashCode() + schema.hashCode()) + (loc ? 1 : 0); - } - - /** */ - public String plan() { - return plan; - } - - /** */ - public String query() { - return qry; - } - - /** */ - public String schema() { - return schema; - } - - /** */ - public boolean local() { - return loc; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - SqlPlanKey key = (SqlPlanKey)o; - - return F.eq(plan, key.plan) && F.eq(qry, key.qry) && F.eq(schema, key.schema) && F.eq(loc, key.loc); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return hash; - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java deleted file mode 100644 index 8cb470cad9ffa..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanValue.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.running; - -/** SQL plan history entry key value. */ -public class SqlPlanValue { - /** Start query timestamp. */ - private final long startTime; - - /** SQL engine. */ - private final SqlPlanHistoryTracker.SqlEngine engine; - - /** - * @param startTime Start query timestamp. - * @param engine Sql engine. - */ - public SqlPlanValue(long startTime, SqlPlanHistoryTracker.SqlEngine engine) { - this.startTime = startTime; - this.engine = engine; - } - - /** */ - public long startTime() { - return startTime; - } - - /** */ - public String engine() { - return engine.name(); - } -} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index dfa92f8ec5dbf..d2b22a1d68de7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -48,6 +48,7 @@ import org.apache.ignite.events.SqlQueryExecutionEvent; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureType; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInternalFuture; @@ -106,7 +107,6 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.processors.query.running.HeavyQueriesTracker; import org.apache.ignite.internal.processors.query.running.RunningQueryManager; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.query.schema.AbstractSchemaChangeListener; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; @@ -456,15 +456,12 @@ private GridQueryFieldsResult executeSelectLocal( ); } - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( + runningQueryManager().planHistoryTracker().addPlan( qryInfo.plan(), qry, qryDesc.schemaName(), true, - qryInfo.beginTs(), - SqlPlanHistoryTracker.SqlEngine.H2 + IndexingQueryEngineConfiguration.ENGINE_NAME ); ResultSet rs = executeWithResumableTimeTracking( @@ -2096,17 +2093,6 @@ private UpdateResult executeUpdate( IndexingQueryFilter filters, GridQueryCancel cancel ) throws IgniteCheckedException { - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( - dml.plan().plan(), - qryDesc.sql(), - qryDesc.schemaName(), - loc, - U.currentTimeMillis(), - SqlPlanHistoryTracker.SqlEngine.H2 - ); - Object[] errKeys = null; long items = 0; @@ -2176,108 +2162,140 @@ private UpdateResult executeUpdate0( IndexingQueryFilter filters, GridQueryCancel cancel ) throws IgniteCheckedException { - UpdatePlan plan = dml.plan(); + StringBuilder dmlPlanInfo = new StringBuilder("As part of this DML command "); - UpdateResult fastUpdateRes = plan.processFast(qryParams.arguments()); + try { + UpdatePlan plan = dml.plan(); - if (fastUpdateRes != null) - return fastUpdateRes; + UpdateResult fastUpdateRes = plan.processFast(qryParams.arguments()); - DmlDistributedPlanInfo distributedPlan = loc ? null : plan.distributedPlan(); + if (fastUpdateRes != null) { + dmlPlanInfo.append("no SELECT queries have been executed."); - if (distributedPlan != null) { - if (cancel == null) - cancel = new GridQueryCancel(); + return fastUpdateRes; + } - UpdateResult result = rdcQryExec.update( - qryDesc.schemaName(), - distributedPlan.getCacheIds(), - qryDesc.sql(), - qryParams.arguments(), - qryDesc.enforceJoinOrder(), - qryParams.pageSize(), - qryParams.timeout(), - qryParams.partitions(), - distributedPlan.isReplicatedOnly(), - cancel - ); + DmlDistributedPlanInfo distributedPlan = loc ? null : plan.distributedPlan(); + + if (distributedPlan != null) { + if (cancel == null) + cancel = new GridQueryCancel(); + + UpdateResult result = rdcQryExec.update( + qryDesc.schemaName(), + distributedPlan.getCacheIds(), + qryDesc.sql(), + qryParams.arguments(), + qryDesc.enforceJoinOrder(), + qryParams.pageSize(), + qryParams.timeout(), + qryParams.partitions(), + distributedPlan.isReplicatedOnly(), + cancel + ); - // Null is returned in case not all nodes support distributed DML. - if (result != null) - return result; - } + // Null is returned in case not all nodes support distributed DML. + if (result != null) { + dmlPlanInfo = new StringBuilder("This DML command has been executed with a distibuted plan " + + "(requests for separate DML commands have been sent to respective cluster nodes)."); - final GridQueryCancel selectCancel = (cancel != null) ? new GridQueryCancel() : null; + return result; + } + } - if (cancel != null) - cancel.add(selectCancel::cancel); - - SqlFieldsQuery selectFieldsQry = new SqlFieldsQuery(plan.selectQuery(), qryDesc.collocated()) - .setArgs(qryParams.arguments()) - .setDistributedJoins(qryDesc.distributedJoins()) - .setEnforceJoinOrder(qryDesc.enforceJoinOrder()) - .setLocal(qryDesc.local()) - .setPageSize(qryParams.pageSize()) - .setTimeout(qryParams.timeout(), TimeUnit.MILLISECONDS) - // We cannot use lazy mode when UPDATE query contains updated columns - // in WHERE condition because it may be cause of update one entry several times - // (when index for such columns is selected for scan): - // e.g. : UPDATE test SET val = val + 1 WHERE val >= ? - .setLazy(qryParams.lazy() && plan.canSelectBeLazy()); - - Iterable> cur; - - // Do a two-step query only if locality flag is not set AND if plan's SELECT corresponds to an actual - // sub-query and not some dummy stuff like "select 1, 2, 3;" - if (!loc && !plan.isLocalSubquery()) { - assert !F.isEmpty(plan.selectQuery()); - - cur = executeSelectForDml( - qryId, - qryDesc.schemaName(), - selectFieldsQry, - selectCancel, - qryParams.timeout() - ); - } - else if (plan.hasRows()) - cur = plan.createRows(qryParams.arguments()); - else { - selectFieldsQry.setLocal(true); + final GridQueryCancel selectCancel = (cancel != null) ? new GridQueryCancel() : null; - QueryParserResult selectParseRes = parser.parse(qryDesc.schemaName(), selectFieldsQry, false); + if (cancel != null) + cancel.add(selectCancel::cancel); - final GridQueryFieldsResult res = executeSelectLocal( - qryId, - selectParseRes.queryDescriptor(), - selectParseRes.queryParameters(), - selectParseRes.select(), - filters, - selectCancel, - qryParams.timeout() - ); + SqlFieldsQuery selectFieldsQry = new SqlFieldsQuery(plan.selectQuery(), qryDesc.collocated()) + .setArgs(qryParams.arguments()) + .setDistributedJoins(qryDesc.distributedJoins()) + .setEnforceJoinOrder(qryDesc.enforceJoinOrder()) + .setLocal(qryDesc.local()) + .setPageSize(qryParams.pageSize()) + .setTimeout(qryParams.timeout(), TimeUnit.MILLISECONDS) + // We cannot use lazy mode when UPDATE query contains updated columns + // in WHERE condition because it may be cause of update one entry several times + // (when index for such columns is selected for scan): + // e.g. : UPDATE test SET val = val + 1 WHERE val >= ? + .setLazy(qryParams.lazy() && plan.canSelectBeLazy()); - cur = new QueryCursorImpl<>(new Iterable>() { - @Override public Iterator> iterator() { - try { - return new GridQueryCacheObjectsIterator(res.iterator(), objectContext(), true); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); + Iterable> cur; + + // Do a two-step query only if locality flag is not set AND if plan's SELECT corresponds to an actual + // sub-query and not some dummy stuff like "select 1, 2, 3;" + if (!loc && !plan.isLocalSubquery()) { + assert !F.isEmpty(plan.selectQuery()); + + cur = executeSelectForDml( + qryId, + qryDesc.schemaName(), + selectFieldsQry, + selectCancel, + qryParams.timeout() + ); + + dmlPlanInfo + .append("the following query has been executed:").append(U.nl()) + .append(selectFieldsQry.getSql()).append(";").append(U.nl()) + .append("check map nodes for map phase query plans"); + } + else if (plan.hasRows()) { + cur = plan.createRows(qryParams.arguments()); + + dmlPlanInfo.append("no SELECT queries have been executed."); + } + else { + selectFieldsQry.setLocal(true); + + QueryParserResult selectParseRes = parser.parse(qryDesc.schemaName(), selectFieldsQry, false); + + final GridQueryFieldsResult res = executeSelectLocal( + qryId, + selectParseRes.queryDescriptor(), + selectParseRes.queryParameters(), + selectParseRes.select(), + filters, + selectCancel, + qryParams.timeout() + ); + + cur = new QueryCursorImpl<>(new Iterable>() { + @Override public Iterator> iterator() { + try { + return new GridQueryCacheObjectsIterator(res.iterator(), objectContext(), true); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } } - } - }, cancel, true, qryParams.lazy()); - } + }, cancel, true, qryParams.lazy()); + + dmlPlanInfo + .append("the following local query has been executed:").append(U.nl()) + .append(selectFieldsQry.getSql()); + } - int pageSize = qryParams.updateBatchSize(); + int pageSize = qryParams.updateBatchSize(); - // TODO: IGNITE-11176 - Need to support cancellation - try { - return DmlUtils.processSelectResult(plan, cur, pageSize); + // TODO: IGNITE-11176 - Need to support cancellation + try { + return DmlUtils.processSelectResult(plan, cur, pageSize); + } + finally { + if (cur instanceof AutoCloseable) + U.closeQuiet((AutoCloseable)cur); + } } finally { - if (cur instanceof AutoCloseable) - U.closeQuiet((AutoCloseable)cur); + runningQueryManager().planHistoryTracker().addPlan( + dmlPlanInfo.toString(), + qryDesc.sql(), + qryDesc.schemaName(), + loc, + IndexingQueryEngineConfiguration.ENGINE_NAME + ); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java index 19f9f8d3fac5c..e9ec4bd7a0df1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java @@ -57,7 +57,7 @@ private DmlArguments() { /** * Value argument. */ - public static class ConstantArgument implements DmlArgument { + private static class ConstantArgument implements DmlArgument { /** Value to return. */ private final Object val; @@ -74,11 +74,6 @@ private ConstantArgument(Object val) { @Override public Object get(Object[] params) { return val; } - - /** */ - public Object getValue() { - return val; - } } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java index dc367e2301e07..9ce39f071530b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java @@ -101,19 +101,4 @@ public UpdateResult execute(GridCacheAdapter cache, Object[] args) throws Ignite return res ? UpdateResult.ONE : UpdateResult.ZERO; } - - /** */ - public DmlArgument keyArg() { - return keyArg; - } - - /** */ - public DmlArgument valArg() { - return valArg; - } - - /** */ - public DmlArgument newValArg() { - return newValArg; - } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 4324ccecd708f..67eae9aadddf3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -18,12 +18,9 @@ package org.apache.ignite.internal.processors.query.h2.dml; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; @@ -534,81 +531,4 @@ public boolean isLocalSubquery() { public boolean canSelectBeLazy() { return canSelectBeLazy; } - - /** - * @return String representation of update plan. - */ - public String plan() { - StringBuilder sb = new StringBuilder("mode=" + mode + - "; table name=" + tbl.cacheName() + - "; group ID=" + tbl.cacheContext().groupId() + - "; columns=" + (colNames == null ? "N/A" : "[" + Arrays.stream(colNames).reduce((a, b) -> a + ", " + b) - .orElse("") + "]") + - "; column types=" + (colTypes == null ? "N/A" : "[" + Arrays.stream(colTypes).mapToObj(Objects::toString) - .reduce((a, b) -> a + ", " + b).orElse("") + "]") + - "; key column index=" + (keyColIdx < 0 ? "N/A" : keyColIdx) + - "; value column index=" + (valColIdx < 0 ? "N/A" : valColIdx) + - "; row count=" + rowsNum + - "; values=" + (rows == null ? "N/A" : "[" + rowsToString(rows) + "]") + - "; select statement based on initial DML statement=" + (selectQry == null ? "N/A" : - ("'" + selectQry + "' (is actual subquery executed on cache=" + isLocSubqry + - ", can be executed in lazy mode=" + canSelectBeLazy + ")"))); - - if (fastUpdate == null) - sb.append("; fast update=N/A"); - else { - List fastUpdateArgs = getFastUpdateArgs(); - - sb.append("; fast update arguments=[key=" + fastUpdateArgs.get(0) + ", value=" + fastUpdateArgs.get(1) + - ", new value=" + fastUpdateArgs.get(2) + "]"); - } - - sb.append("; distributed plan info="); - - if (distributed == null) - sb.append("N/A"); - else { - sb.append("[IDs of caches involved in update=[" + - distributed.getCacheIds().stream().map(Object::toString).collect(Collectors.joining(", ")) + "]" + - ", update involves only replicated caches=" + distributed.isReplicatedOnly()); - } - - return sb.toString(); - } - - /** - * @param rows Rows. - */ - public String rowsToString(List> rows) { - StringBuilder result = new StringBuilder(); - - for (List row : rows) { - if (!row.isEmpty()) { - result.append("{"); - - result.append(row.stream() - .map(arg -> ((DmlArguments.ConstantArgument)arg).getValue().toString()) - .collect(Collectors.joining(", "))); - - result.append("}, "); - } - } - - return result.length() > 0 ? result.substring(0, result.length() - 2) : ""; - } - - /** */ - public List getFastUpdateArgs() { - assert fastUpdate != null; - - DmlArguments.ConstantArgument keyArg = (DmlArguments.ConstantArgument)fastUpdate.keyArg(); - DmlArguments.ConstantArgument valArg = (DmlArguments.ConstantArgument)fastUpdate.valArg(); - DmlArguments.ConstantArgument newValArg = (DmlArguments.ConstantArgument)fastUpdate.newValArg(); - - return Arrays.asList( - keyArg.getValue() == null ? "NULL" : keyArg.getValue().toString(), - valArg.getValue() == null ? "NULL" : valArg.getValue().toString(), - newValArg.getValue() == null ? "NULL" : newValArg.getValue().toString() - ); - } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 47c7b949dee14..e6ff5ecbfe6e2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -42,6 +42,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.metric.IoStatisticsHolder; @@ -69,7 +70,6 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; import org.apache.ignite.internal.processors.tracing.Span; @@ -477,15 +477,12 @@ private void onQueryRequest0( ); } - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( + h2.runningQueryManager().planHistoryTracker().addPlan( qryInfo.plan(), sql, schemaName, false, - qryInfo.beginTs(), - SqlPlanHistoryTracker.SqlEngine.H2 + IndexingQueryEngineConfiguration.ENGINE_NAME ); GridQueryCancel qryCancel = qryResults.queryCancel(qryIdx); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 0cde8ca1b2c69..8427165616928 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -47,6 +47,7 @@ import org.apache.ignite.cache.query.QueryRetryException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -77,7 +78,6 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; -import org.apache.ignite.internal.processors.query.running.SqlPlanHistoryTracker; import org.apache.ignite.internal.processors.tracing.MTC; import org.apache.ignite.internal.processors.tracing.MTC.TraceSurroundings; import org.apache.ignite.internal.util.typedef.C2; @@ -527,15 +527,12 @@ else if (QueryUtils.wasCancelled(err)) ); } - SqlPlanHistoryTracker planHistTracker = ctx.query().runningQueryManager().planHistoryTracker(); - - planHistTracker.addPlan( + h2.runningQueryManager().planHistoryTracker().addPlan( qryInfo.plan(), qry.originalSql(), schemaName, qry.isLocal(), - qryStartTime, - SqlPlanHistoryTracker.SqlEngine.H2 + IndexingQueryEngineConfiguration.ENGINE_NAME ); H2PooledConnection conn0 = conn; From febb24f9047dbfab6d54880526dbbf5f46473d37 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Tue, 10 Sep 2024 22:41:44 +1000 Subject: [PATCH 16/33] IGNITE-22557 Minor tests refactoring --- .../integration/SqlPlanHistoryClientIntegrationTest.java | 1 + .../SqlPlanHistoryH2ClientIntegrationTest.java | 1 + .../integration/SqlPlanHistoryIntegrationTest.java | 8 +++++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java index 7734a068b29d0..2c7cbbe574801 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java @@ -41,6 +41,7 @@ public class SqlPlanHistoryClientIntegrationTest extends SqlPlanHistoryIntegrati /** {@inheritDoc} */ @Override protected void startTestGrid() throws Exception { startGrids(2); + startClientGrid(2); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java index a951c995c00d9..51a1aedc2ed17 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java @@ -41,6 +41,7 @@ public class SqlPlanHistoryH2ClientIntegrationTest extends SqlPlanHistoryH2Integ /** {@inheritDoc} */ @Override protected void startTestGrid() throws Exception { startGrids(2); + startClientGrid(2); } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index f00ffcc5169cf..0e1b2510eddd9 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -536,6 +536,7 @@ public void cacheQuery(Query qry, String cacheName) { /** * @param cmds Set of DML commands. + * @param isSimpleQry Simple query flag. */ public void runJdbcDml(List cmds, boolean isSimpleQry) throws SQLException { executeJdbcDml((stmt) -> { @@ -552,6 +553,7 @@ public void runJdbcDml(List cmds, boolean isSimpleQry) throws SQLExcepti /** * @param task Task to execute. + * @param isSimpleQry Simple query flag. */ public void executeJdbcDml(Consumer task, boolean isSimpleQry) throws SQLException { if (loc || !isFullyFetched) @@ -569,6 +571,7 @@ public void executeJdbcDml(Consumer task, boolean isSimpleQry) throws /** * @param cmds Set of DML commands. + * @param isSimpleQry Simple query flag. */ public void runSqlFieldsQueryDml(List cmds, boolean isSimpleQry) { executeSqlFieldsQueryDml( @@ -578,6 +581,7 @@ public void runSqlFieldsQueryDml(List cmds, boolean isSimpleQry) { /** * @param task Task to execute. + * @param isSimpleQry Simple query flag. */ public void executeSqlFieldsQueryDml(Consumer> task, boolean isSimpleQry) { if (isClient && loc || !isFullyFetched) @@ -590,7 +594,9 @@ public void executeSqlFieldsQueryDml(Consumer> task checkSqlPlanHistoryDml(3, isSimpleQry); } - /** */ + /** + * @param node Ignite node to check SQL plan history for. + */ public Collection getSqlPlanHistory(IgniteEx node) { return node.context().query().runningQueryManager().planHistoryTracker() .sqlPlanHistory().keySet(); From 16672945d728a488c9c2adbc25161d1d98247c27 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 13:05:20 +1000 Subject: [PATCH 17/33] IGNITE-22557 GridBoundedConcurrentLinkedHashMap changed to GridBoundedConcurrentLinkedHashSet, SqlPlanHistoryTracker#addPlan changed, SqlPlanHistoryIntegrationTest#testEntryReplacement added --- .../SqlPlanHistoryIntegrationTest.java | 29 +++++++++++++++++-- .../query/running/RunningQueryManager.java | 2 +- .../query/running/SqlPlanHistoryTracker.java | 19 +++++++----- .../processors/query/h2/dml/UpdatePlan.java | 4 +-- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 0e1b2510eddd9..6816500511e84 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -436,6 +436,32 @@ public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { assertFalse(qrys.contains(SQL + " where A.fail=" + i)); } + /** + * Checks that older SQL plan history entries are replaced with newer ones with the same parameters (except for + * the beginning time). + */ + @Test + public void testEntryReplacement() throws InterruptedException { + if (loc || !isFullyFetched || (isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME)) + return; + + long[] timeStamps = new long[2]; + + for (int i = 0; i < 2; i++) { + executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); + + Collection plans = getSqlPlanHistory(queryNode()); + + assertEquals(1, plans.size()); + + timeStamps[i] = plans.stream().findFirst().get().startTime(); + + Thread.sleep(10); + } + + assertTrue(timeStamps[1] > timeStamps[0]); + } + /** Checks that SQL plan history remains empty if history size is set to zero. */ @Test public void testEmptyPlanHistory() { @@ -598,8 +624,7 @@ public void executeSqlFieldsQueryDml(Consumer> task * @param node Ignite node to check SQL plan history for. */ public Collection getSqlPlanHistory(IgniteEx node) { - return node.context().query().runningQueryManager().planHistoryTracker() - .sqlPlanHistory().keySet(); + return new ArrayList<>(node.context().query().runningQueryManager().planHistoryTracker().sqlPlanHistory()); } /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java index 9ddb16b63c5dc..96a216c6da90e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java @@ -215,7 +215,7 @@ public RunningQueryManager(GridKernalContext ctx) { ctx.systemView().registerView(SQL_PLAN_HIST_VIEW, SQL_PLAN_HIST_VIEW_DESC, new SqlPlanHistoryViewWalker(), - planHistTracker.sqlPlanHistory().keySet(), + new ArrayList<>(planHistTracker.sqlPlanHistory()), SqlPlanHistoryView::new); MetricRegistryImpl userMetrics = ctx.metric().registry(SQL_USER_QUERIES_REG_NAME); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index fdbc237b915a9..b5ec9b7cd6a64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -18,13 +18,13 @@ package org.apache.ignite.internal.processors.query.running; import java.util.Collections; -import java.util.Map; -import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; +import java.util.Set; +import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet; /** Class that manages recording and storing SQL plans. */ public class SqlPlanHistoryTracker { /** SQL plan history. */ - private final GridBoundedConcurrentLinkedHashMap sqlPlanHistory; + private final GridBoundedConcurrentLinkedHashSet sqlPlanHistory; /** SQL plan history size. */ private int historySize; @@ -35,7 +35,7 @@ public class SqlPlanHistoryTracker { public SqlPlanHistoryTracker(int historySize) { this.historySize = historySize; - sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashMap<>(historySize) : null; + sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashSet<>(historySize) : null; } /** @@ -51,15 +51,18 @@ public void addPlan(String plan, String qry, String schema, boolean loc, String SqlPlan sqlPlan = new SqlPlan(plan, qry, schema, loc, engine); - sqlPlanHistory.put(sqlPlan, sqlPlan.startTime()); + if (sqlPlanHistory.contains(sqlPlan)) + sqlPlanHistory.remove(sqlPlan); + + sqlPlanHistory.add(sqlPlan); } /** */ - public Map sqlPlanHistory() { + public Set sqlPlanHistory() { if (historySize <= 0) - return Collections.emptyMap(); + return Collections.emptySet(); - return Collections.unmodifiableMap(sqlPlanHistory); + return Collections.unmodifiableSet(sqlPlanHistory); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 67eae9aadddf3..d93b33bab0b24 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -466,9 +466,9 @@ private void extractArgsValues(Object[] args, List> res, GridQueryRowDes for (int j = 0; j < colNames.length; j++) { Object colVal; if (fillAbsentPKsWithDefaults) - colVal = row.size() > j ? row.get(j).get(args) : null; + colVal = row.size() > j ? row.get(j).get(args) : null; else - colVal = row.get(j).get(args); + colVal = row.get(j).get(args); if (j == keyColIdx || j == valColIdx) { Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); From 8a437a22e226b841ea7b841118eab98d742f1dad Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 13:12:49 +1000 Subject: [PATCH 18/33] IGNITE-22557 Whitespaces refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index d93b33bab0b24..529de7d594f77 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -466,7 +466,7 @@ private void extractArgsValues(Object[] args, List> res, GridQueryRowDes for (int j = 0; j < colNames.length; j++) { Object colVal; if (fillAbsentPKsWithDefaults) - colVal = row.size() > j ? row.get(j).get(args) : null; + colVal = row.size() > j ? row.get(j).get(args) : null; else colVal = row.get(j).get(args); From 886c76323e019cb39f24fd3ca5ebc1ac264003c2 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 13:14:56 +1000 Subject: [PATCH 19/33] IGNITE-22557 Whitespaces refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 529de7d594f77..67eae9aadddf3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -468,7 +468,7 @@ private void extractArgsValues(Object[] args, List> res, GridQueryRowDes if (fillAbsentPKsWithDefaults) colVal = row.size() > j ? row.get(j).get(args) : null; else - colVal = row.get(j).get(args); + colVal = row.get(j).get(args); if (j == keyColIdx || j == valColIdx) { Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); From 657b86b8f235ce4177a836ff580219710d8284de Mon Sep 17 00:00:00 2001 From: oleg-vlsk <153691984+oleg-vlsk@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:18:19 +1000 Subject: [PATCH 20/33] Delete changes in UpdatePlan --- .../processors/query/h2/dml/UpdatePlan.java | 534 ------------------ 1 file changed, 534 deletions(-) delete mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java deleted file mode 100644 index 67eae9aadddf3..0000000000000 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ /dev/null @@ -1,534 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.dml; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.binary.BinaryObject; -import org.apache.ignite.binary.BinaryObjectBuilder; -import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; -import org.apache.ignite.internal.processors.query.GridQueryProperty; -import org.apache.ignite.internal.processors.query.GridQueryRowDescriptor; -import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; -import org.apache.ignite.internal.processors.query.IgniteSQLException; -import org.apache.ignite.internal.processors.query.QueryUtils; -import org.apache.ignite.internal.processors.query.h2.UpdateResult; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T3; -import org.apache.ignite.lang.IgniteBiTuple; -import org.h2.table.Column; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.internal.processors.query.h2.dml.UpdateMode.BULK_LOAD; - -/** - * Update plan - where to take data to update cache from and how to construct new keys and values, if needed. - */ -public final class UpdatePlan { - /** Initial statement to drive the rest of the logic. */ - private final UpdateMode mode; - - /** to be affected by initial DML statement. */ - private final GridH2Table tbl; - - /** Column names to set or update. */ - private final String[] colNames; - - /** Column types to set for insert/merge. */ - private final int[] colTypes; - - /** Method to create key for INSERT or MERGE, ignored for UPDATE and DELETE. */ - private final KeyValueSupplier keySupplier; - - /** Method to create value to put to cache, ignored for DELETE. */ - private final KeyValueSupplier valSupplier; - - /** Key column index. */ - private final int keyColIdx; - - /** Value column index. */ - private final int valColIdx; - - /** SELECT statement built upon initial DML statement. */ - private final String selectQry; - - /** Subquery flag - {@code true} if {@link #selectQry} is an actual subquery that retrieves data from some cache. */ - private final boolean isLocSubqry; - - /** Rows for query-less MERGE or INSERT. */ - private final List> rows; - - /** Number of rows in rows based MERGE or INSERT. */ - private final int rowsNum; - - /** Whether absent PK parts should be filled with defaults or not. */ - private boolean fillAbsentPKsWithDefaults; - - /** Arguments for fast UPDATE or DELETE. */ - private final FastUpdate fastUpdate; - - /** Additional info for distributed update. */ - private final DmlDistributedPlanInfo distributed; - - /** Additional info for distributed update. */ - private final boolean canSelectBeLazy; - - /** - * Constructor. - * - * @param mode Mode. - * @param tbl Table. - * @param colNames Column names. - * @param colTypes Column types. - * @param keySupplier Key supplier. - * @param valSupplier Value supplier. - * @param keyColIdx Key column index. - * @param valColIdx value column index. - * @param selectQry Select query. - * @param isLocSubqry Local subquery flag. - * @param rows Rows for query-less INSERT or MERGE. - * @param rowsNum Rows number. - * @param fastUpdate Fast update (if any). - * @param distributed Distributed plan (if any) - * @param fillAbsentPKsWithDefaults Fills absent PKs with nulls or defaults setting. - */ - public UpdatePlan( - UpdateMode mode, - GridH2Table tbl, - String[] colNames, - int[] colTypes, - KeyValueSupplier keySupplier, - KeyValueSupplier valSupplier, - int keyColIdx, - int valColIdx, - String selectQry, - boolean isLocSubqry, - List> rows, - int rowsNum, - @Nullable FastUpdate fastUpdate, - @Nullable DmlDistributedPlanInfo distributed, - boolean canSelectBeLazy, - boolean fillAbsentPKsWithDefaults - ) { - this.colNames = colNames; - this.colTypes = colTypes; - this.rows = rows; - this.rowsNum = rowsNum; - this.fillAbsentPKsWithDefaults = fillAbsentPKsWithDefaults; - - assert mode != null; - assert tbl != null; - - this.mode = mode; - this.tbl = tbl; - this.keySupplier = keySupplier; - this.valSupplier = valSupplier; - this.keyColIdx = keyColIdx; - this.valColIdx = valColIdx; - this.selectQry = selectQry; - this.isLocSubqry = isLocSubqry; - this.fastUpdate = fastUpdate; - this.distributed = distributed; - this.canSelectBeLazy = canSelectBeLazy; - } - - /** - * Constructor for delete operation or fast update. - * - * @param mode Mode. - * @param tbl Table. - * @param selectQry Select query. - * @param fastUpdate Fast update arguments (if any). - * @param distributed Distributed plan (if any) - */ - public UpdatePlan( - UpdateMode mode, - GridH2Table tbl, - String selectQry, - @Nullable FastUpdate fastUpdate, - @Nullable DmlDistributedPlanInfo distributed - ) { - this( - mode, - tbl, - null, - null, - null, - null, - -1, - -1, - selectQry, - false, - null, - 0, - fastUpdate, - distributed, - true, - false - ); - } - - /** - * Convert a row into key-value pair. - * - * @param row Row to process. - * @throws IgniteCheckedException if failed. - */ - public IgniteBiTuple processRow(List row) throws IgniteCheckedException { - if (mode != BULK_LOAD && row.size() != colNames.length) - throw new IgniteSQLException("Not enough values in a row: " + row.size() + " instead of " + colNames.length, - IgniteQueryErrorCode.ENTRY_PROCESSING); - - GridQueryRowDescriptor rowDesc = tbl.rowDescriptor(); - GridQueryTypeDescriptor desc = rowDesc.type(); - - GridCacheContext cctx = rowDesc.context(); - - Object key = keySupplier.apply(row); - - if (QueryUtils.isSqlType(desc.keyClass())) { - assert keyColIdx != -1; - - key = DmlUtils.convert(key, rowDesc, desc.keyClass(), colTypes[keyColIdx], colNames[keyColIdx]); - } - - Object val = valSupplier.apply(row); - - if (QueryUtils.isSqlType(desc.valueClass())) { - assert valColIdx != -1; - - val = DmlUtils.convert(val, rowDesc, desc.valueClass(), colTypes[valColIdx], colNames[valColIdx]); - } - - if (key == null) { - if (F.isEmpty(desc.keyFieldName())) - throw new IgniteSQLException("Key for INSERT, COPY, or MERGE must not be null", - IgniteQueryErrorCode.NULL_KEY); - else - throw new IgniteSQLException("Null value is not allowed for column '" + desc.keyFieldName() + "'", - IgniteQueryErrorCode.NULL_KEY); - } - - if (val == null) { - if (F.isEmpty(desc.valueFieldName())) - throw new IgniteSQLException("Value for INSERT, COPY, MERGE, or UPDATE must not be null", - IgniteQueryErrorCode.NULL_VALUE); - else - throw new IgniteSQLException("Null value is not allowed for column '" + desc.valueFieldName() + "'", - IgniteQueryErrorCode.NULL_VALUE); - } - - int actualColCnt = Math.min(colNames.length, row.size()); - - Map newColVals = new HashMap<>(); - - for (int i = 0; i < actualColCnt; i++) { - if (i == keyColIdx || i == valColIdx) - continue; - - String colName = colNames[i]; - - GridQueryProperty prop = desc.property(colName); - - assert prop != null; - - Class expCls = prop.type(); - - newColVals.put(colName, DmlUtils.convert(row.get(i), rowDesc, expCls, colTypes[i], colNames[i])); - } - - desc.setDefaults(key, val); - - // We update columns in the order specified by the table for a reason - table's - // column order preserves their precedence for correct update of nested properties. - Column[] tblCols = tbl.getColumns(); - - // First 2 columns are _key and _val Skip 'em. - for (int i = QueryUtils.DEFAULT_COLUMNS_COUNT; i < tblCols.length; i++) { - if (tbl.rowDescriptor().isKeyColumn(i) || tbl.rowDescriptor().isValueColumn(i)) - continue; - - String colName = tblCols[i].getName(); - - if (!newColVals.containsKey(colName)) - continue; - - Object colVal = newColVals.get(colName); - - desc.setValue(colName, key, val, colVal); - } - - if (cctx.binaryMarshaller()) { - if (key instanceof BinaryObjectBuilder) - key = ((BinaryObjectBuilder)key).build(); - - if (val instanceof BinaryObjectBuilder) - val = ((BinaryObjectBuilder)val).build(); - } - - desc.validateKeyAndValue(key, val); - - return new IgniteBiTuple<>(key, val); - } - - /** - * Convert a row into value. - * - * @param row Row to process. - * @throws IgniteCheckedException if failed. - * @return Tuple contains: [key, old value, new value] - */ - public T3 processRowForUpdate(List row) throws IgniteCheckedException { - GridQueryRowDescriptor rowDesc = tbl.rowDescriptor(); - GridQueryTypeDescriptor desc = rowDesc.type(); - - GridCacheContext cctx = rowDesc.context(); - - boolean hasNewVal = (valColIdx != -1); - - boolean hasProps = !hasNewVal || colNames.length > 1; - - Object key = row.get(0); - - Object oldVal = row.get(1); - - if (cctx.binaryMarshaller() && !(oldVal instanceof BinaryObject)) - oldVal = cctx.grid().binary().toBinary(oldVal); - - Object newVal; - - Map newColVals = new HashMap<>(); - - for (int i = 0; i < colNames.length; i++) { - if (hasNewVal && i == valColIdx - 2) - continue; - - GridQueryProperty prop = tbl.rowDescriptor().type().property(colNames[i]); - - assert prop != null : "Unknown property: " + colNames[i]; - - newColVals.put(colNames[i], DmlUtils.convert(row.get(i + 2), rowDesc, prop.type(), colTypes[i], colNames[i])); - } - - newVal = valSupplier.apply(row); - - if (newVal == null) - throw new IgniteSQLException("New value for UPDATE must not be null", IgniteQueryErrorCode.NULL_VALUE); - - // Skip key and value - that's why we start off with 3rd column - for (int i = 0; i < tbl.getColumns().length - QueryUtils.DEFAULT_COLUMNS_COUNT; i++) { - Column c = tbl.getColumn(i + QueryUtils.DEFAULT_COLUMNS_COUNT); - - if (rowDesc.isKeyColumn(c.getColumnId()) || rowDesc.isValueColumn(c.getColumnId())) - continue; - - GridQueryProperty prop = desc.property(c.getName()); - - if (prop.key()) - continue; // Don't get values of key's columns - we won't use them anyway - - boolean hasNewColVal = newColVals.containsKey(c.getName()); - - if (!hasNewColVal) - continue; - - Object colVal = newColVals.get(c.getName()); - - // UPDATE currently does not allow to modify key or its fields, so we must be safe to pass null as key. - rowDesc.setFieldValue(null, newVal, colVal, i); - } - - if (cctx.binaryMarshaller() && hasProps) { - assert newVal instanceof BinaryObjectBuilder; - - newVal = ((BinaryObjectBuilder)newVal).build(); - } - - desc.validateKeyAndValue(key, newVal); - - return new T3<>(key, oldVal, newVal); - } - - /** - * Process fast DML operation if possible. - * - * @param args QUery arguments. - * @return Update result or {@code null} if fast update is not applicable for plan. - * @throws IgniteCheckedException If failed. - */ - public UpdateResult processFast(Object[] args) throws IgniteCheckedException { - if (fastUpdate != null) - return fastUpdate.execute(cacheContext().cache(), args); - - return null; - } - - /** - * @return {@code True} if predefined rows exist. - */ - public boolean hasRows() { - return !F.isEmpty(rows); - } - - /** - * Extract rows from plan without performing any query. - * - * @param args Original query arguments. - * @return {@link List} of rows from the plan for a single query. - * For example, if we have multiple args in a query:
- * {@code INSERT INTO person VALUES (k1, v1), (k2, v2), (k3, v3);}
- * we will get a {@link List} of {@link List} with items {@code {[k1, v1], [k2, v2], [k3, v3]}}. - * @throws IgniteCheckedException if failed. - */ - public List> createRows(Object[] args) throws IgniteCheckedException { - assert rowsNum > 0 && !F.isEmpty(colNames); - - List> res = new ArrayList<>(rowsNum); - - GridQueryRowDescriptor desc = tbl.rowDescriptor(); - - extractArgsValues(args, res, desc); - - return res; - } - - /** - * Extract rows from plan without performing any query. - * - * @param argss Batch of arguments. - * @return {@link List} of rows from the plan for each query. - * For example, if we have a batch of queries with multiple args:
- * - * INSERT INTO person VALUES (k1, v1), (k2, v2), (k3, v3);
- * INSERT INTO person VALUES (k4, v4), (k5, v5), (k6, v6);
- * - * we will get a {@link List} of {@link List} of {@link List} with items:
- * - * {[k1, v1], [k2, v2], [k3, v3]},
- * {[k4, v4], [k5, v5], [k6, v6]}
- * - * @throws IgniteCheckedException If failed. - */ - public List>> createRows(List argss) throws IgniteCheckedException { - assert rowsNum > 0 && !F.isEmpty(colNames); - assert argss != null; - - List>> resPerQry = new ArrayList<>(argss.size()); - - GridQueryRowDescriptor desc = tbl.rowDescriptor(); - - for (Object[] args : argss) { - List> res = new ArrayList<>(); - - resPerQry.add(res); - - extractArgsValues(args, res, desc); - } - - return resPerQry; - } - - /** - * Extracts values from arguments. - * - * @param args Arguments. - * @param res Result list where to put values to. - * @param desc Row descriptor. - * @throws IgniteCheckedException If failed. - */ - private void extractArgsValues(Object[] args, List> res, GridQueryRowDescriptor desc) - throws IgniteCheckedException { - assert res != null; - - for (List row : rows) { - List resRow = new ArrayList<>(); - - for (int j = 0; j < colNames.length; j++) { - Object colVal; - if (fillAbsentPKsWithDefaults) - colVal = row.size() > j ? row.get(j).get(args) : null; - else - colVal = row.get(j).get(args); - - if (j == keyColIdx || j == valColIdx) { - Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); - - colVal = DmlUtils.convert(colVal, desc, colCls, colTypes[j], colNames[j]); - } - - resRow.add(colVal); - } - - res.add(resRow); - } - } - - /** - * @return Update mode. - */ - public UpdateMode mode() { - return mode; - } - - /** - * @return Cache context. - */ - public GridCacheContext cacheContext() { - return tbl.cacheContext(); - } - - /** - * @return Distributed plan info (for skip-reducer mode). - */ - @Nullable public DmlDistributedPlanInfo distributedPlan() { - return distributed; - } - - /** - * @return Row count. - */ - public int rowCount() { - return rowsNum; - } - - /** - * @return Select query. - */ - public String selectQuery() { - return selectQry; - } - - /** - * @return Local subquery flag. - */ - public boolean isLocalSubquery() { - return isLocSubqry; - } - - /** - * @return {@code true} is the SELECT query may be executed in lazy mode. - */ - public boolean canSelectBeLazy() { - return canSelectBeLazy; - } -} From be4551ab2d89175a105aff3fdd2a47e1fed17245 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 13:56:25 +1000 Subject: [PATCH 21/33] Revert "Delete changes in UpdatePlan" This reverts commit fb2a47cd984d70e30995a3716ca86f3175267d02. --- .../processors/query/h2/dml/UpdatePlan.java | 534 ++++++++++++++++++ 1 file changed, 534 insertions(+) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java new file mode 100644 index 0000000000000..67eae9aadddf3 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -0,0 +1,534 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.dml; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; +import org.apache.ignite.internal.processors.query.GridQueryProperty; +import org.apache.ignite.internal.processors.query.GridQueryRowDescriptor; +import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.processors.query.QueryUtils; +import org.apache.ignite.internal.processors.query.h2.UpdateResult; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T3; +import org.apache.ignite.lang.IgniteBiTuple; +import org.h2.table.Column; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.query.h2.dml.UpdateMode.BULK_LOAD; + +/** + * Update plan - where to take data to update cache from and how to construct new keys and values, if needed. + */ +public final class UpdatePlan { + /** Initial statement to drive the rest of the logic. */ + private final UpdateMode mode; + + /** to be affected by initial DML statement. */ + private final GridH2Table tbl; + + /** Column names to set or update. */ + private final String[] colNames; + + /** Column types to set for insert/merge. */ + private final int[] colTypes; + + /** Method to create key for INSERT or MERGE, ignored for UPDATE and DELETE. */ + private final KeyValueSupplier keySupplier; + + /** Method to create value to put to cache, ignored for DELETE. */ + private final KeyValueSupplier valSupplier; + + /** Key column index. */ + private final int keyColIdx; + + /** Value column index. */ + private final int valColIdx; + + /** SELECT statement built upon initial DML statement. */ + private final String selectQry; + + /** Subquery flag - {@code true} if {@link #selectQry} is an actual subquery that retrieves data from some cache. */ + private final boolean isLocSubqry; + + /** Rows for query-less MERGE or INSERT. */ + private final List> rows; + + /** Number of rows in rows based MERGE or INSERT. */ + private final int rowsNum; + + /** Whether absent PK parts should be filled with defaults or not. */ + private boolean fillAbsentPKsWithDefaults; + + /** Arguments for fast UPDATE or DELETE. */ + private final FastUpdate fastUpdate; + + /** Additional info for distributed update. */ + private final DmlDistributedPlanInfo distributed; + + /** Additional info for distributed update. */ + private final boolean canSelectBeLazy; + + /** + * Constructor. + * + * @param mode Mode. + * @param tbl Table. + * @param colNames Column names. + * @param colTypes Column types. + * @param keySupplier Key supplier. + * @param valSupplier Value supplier. + * @param keyColIdx Key column index. + * @param valColIdx value column index. + * @param selectQry Select query. + * @param isLocSubqry Local subquery flag. + * @param rows Rows for query-less INSERT or MERGE. + * @param rowsNum Rows number. + * @param fastUpdate Fast update (if any). + * @param distributed Distributed plan (if any) + * @param fillAbsentPKsWithDefaults Fills absent PKs with nulls or defaults setting. + */ + public UpdatePlan( + UpdateMode mode, + GridH2Table tbl, + String[] colNames, + int[] colTypes, + KeyValueSupplier keySupplier, + KeyValueSupplier valSupplier, + int keyColIdx, + int valColIdx, + String selectQry, + boolean isLocSubqry, + List> rows, + int rowsNum, + @Nullable FastUpdate fastUpdate, + @Nullable DmlDistributedPlanInfo distributed, + boolean canSelectBeLazy, + boolean fillAbsentPKsWithDefaults + ) { + this.colNames = colNames; + this.colTypes = colTypes; + this.rows = rows; + this.rowsNum = rowsNum; + this.fillAbsentPKsWithDefaults = fillAbsentPKsWithDefaults; + + assert mode != null; + assert tbl != null; + + this.mode = mode; + this.tbl = tbl; + this.keySupplier = keySupplier; + this.valSupplier = valSupplier; + this.keyColIdx = keyColIdx; + this.valColIdx = valColIdx; + this.selectQry = selectQry; + this.isLocSubqry = isLocSubqry; + this.fastUpdate = fastUpdate; + this.distributed = distributed; + this.canSelectBeLazy = canSelectBeLazy; + } + + /** + * Constructor for delete operation or fast update. + * + * @param mode Mode. + * @param tbl Table. + * @param selectQry Select query. + * @param fastUpdate Fast update arguments (if any). + * @param distributed Distributed plan (if any) + */ + public UpdatePlan( + UpdateMode mode, + GridH2Table tbl, + String selectQry, + @Nullable FastUpdate fastUpdate, + @Nullable DmlDistributedPlanInfo distributed + ) { + this( + mode, + tbl, + null, + null, + null, + null, + -1, + -1, + selectQry, + false, + null, + 0, + fastUpdate, + distributed, + true, + false + ); + } + + /** + * Convert a row into key-value pair. + * + * @param row Row to process. + * @throws IgniteCheckedException if failed. + */ + public IgniteBiTuple processRow(List row) throws IgniteCheckedException { + if (mode != BULK_LOAD && row.size() != colNames.length) + throw new IgniteSQLException("Not enough values in a row: " + row.size() + " instead of " + colNames.length, + IgniteQueryErrorCode.ENTRY_PROCESSING); + + GridQueryRowDescriptor rowDesc = tbl.rowDescriptor(); + GridQueryTypeDescriptor desc = rowDesc.type(); + + GridCacheContext cctx = rowDesc.context(); + + Object key = keySupplier.apply(row); + + if (QueryUtils.isSqlType(desc.keyClass())) { + assert keyColIdx != -1; + + key = DmlUtils.convert(key, rowDesc, desc.keyClass(), colTypes[keyColIdx], colNames[keyColIdx]); + } + + Object val = valSupplier.apply(row); + + if (QueryUtils.isSqlType(desc.valueClass())) { + assert valColIdx != -1; + + val = DmlUtils.convert(val, rowDesc, desc.valueClass(), colTypes[valColIdx], colNames[valColIdx]); + } + + if (key == null) { + if (F.isEmpty(desc.keyFieldName())) + throw new IgniteSQLException("Key for INSERT, COPY, or MERGE must not be null", + IgniteQueryErrorCode.NULL_KEY); + else + throw new IgniteSQLException("Null value is not allowed for column '" + desc.keyFieldName() + "'", + IgniteQueryErrorCode.NULL_KEY); + } + + if (val == null) { + if (F.isEmpty(desc.valueFieldName())) + throw new IgniteSQLException("Value for INSERT, COPY, MERGE, or UPDATE must not be null", + IgniteQueryErrorCode.NULL_VALUE); + else + throw new IgniteSQLException("Null value is not allowed for column '" + desc.valueFieldName() + "'", + IgniteQueryErrorCode.NULL_VALUE); + } + + int actualColCnt = Math.min(colNames.length, row.size()); + + Map newColVals = new HashMap<>(); + + for (int i = 0; i < actualColCnt; i++) { + if (i == keyColIdx || i == valColIdx) + continue; + + String colName = colNames[i]; + + GridQueryProperty prop = desc.property(colName); + + assert prop != null; + + Class expCls = prop.type(); + + newColVals.put(colName, DmlUtils.convert(row.get(i), rowDesc, expCls, colTypes[i], colNames[i])); + } + + desc.setDefaults(key, val); + + // We update columns in the order specified by the table for a reason - table's + // column order preserves their precedence for correct update of nested properties. + Column[] tblCols = tbl.getColumns(); + + // First 2 columns are _key and _val Skip 'em. + for (int i = QueryUtils.DEFAULT_COLUMNS_COUNT; i < tblCols.length; i++) { + if (tbl.rowDescriptor().isKeyColumn(i) || tbl.rowDescriptor().isValueColumn(i)) + continue; + + String colName = tblCols[i].getName(); + + if (!newColVals.containsKey(colName)) + continue; + + Object colVal = newColVals.get(colName); + + desc.setValue(colName, key, val, colVal); + } + + if (cctx.binaryMarshaller()) { + if (key instanceof BinaryObjectBuilder) + key = ((BinaryObjectBuilder)key).build(); + + if (val instanceof BinaryObjectBuilder) + val = ((BinaryObjectBuilder)val).build(); + } + + desc.validateKeyAndValue(key, val); + + return new IgniteBiTuple<>(key, val); + } + + /** + * Convert a row into value. + * + * @param row Row to process. + * @throws IgniteCheckedException if failed. + * @return Tuple contains: [key, old value, new value] + */ + public T3 processRowForUpdate(List row) throws IgniteCheckedException { + GridQueryRowDescriptor rowDesc = tbl.rowDescriptor(); + GridQueryTypeDescriptor desc = rowDesc.type(); + + GridCacheContext cctx = rowDesc.context(); + + boolean hasNewVal = (valColIdx != -1); + + boolean hasProps = !hasNewVal || colNames.length > 1; + + Object key = row.get(0); + + Object oldVal = row.get(1); + + if (cctx.binaryMarshaller() && !(oldVal instanceof BinaryObject)) + oldVal = cctx.grid().binary().toBinary(oldVal); + + Object newVal; + + Map newColVals = new HashMap<>(); + + for (int i = 0; i < colNames.length; i++) { + if (hasNewVal && i == valColIdx - 2) + continue; + + GridQueryProperty prop = tbl.rowDescriptor().type().property(colNames[i]); + + assert prop != null : "Unknown property: " + colNames[i]; + + newColVals.put(colNames[i], DmlUtils.convert(row.get(i + 2), rowDesc, prop.type(), colTypes[i], colNames[i])); + } + + newVal = valSupplier.apply(row); + + if (newVal == null) + throw new IgniteSQLException("New value for UPDATE must not be null", IgniteQueryErrorCode.NULL_VALUE); + + // Skip key and value - that's why we start off with 3rd column + for (int i = 0; i < tbl.getColumns().length - QueryUtils.DEFAULT_COLUMNS_COUNT; i++) { + Column c = tbl.getColumn(i + QueryUtils.DEFAULT_COLUMNS_COUNT); + + if (rowDesc.isKeyColumn(c.getColumnId()) || rowDesc.isValueColumn(c.getColumnId())) + continue; + + GridQueryProperty prop = desc.property(c.getName()); + + if (prop.key()) + continue; // Don't get values of key's columns - we won't use them anyway + + boolean hasNewColVal = newColVals.containsKey(c.getName()); + + if (!hasNewColVal) + continue; + + Object colVal = newColVals.get(c.getName()); + + // UPDATE currently does not allow to modify key or its fields, so we must be safe to pass null as key. + rowDesc.setFieldValue(null, newVal, colVal, i); + } + + if (cctx.binaryMarshaller() && hasProps) { + assert newVal instanceof BinaryObjectBuilder; + + newVal = ((BinaryObjectBuilder)newVal).build(); + } + + desc.validateKeyAndValue(key, newVal); + + return new T3<>(key, oldVal, newVal); + } + + /** + * Process fast DML operation if possible. + * + * @param args QUery arguments. + * @return Update result or {@code null} if fast update is not applicable for plan. + * @throws IgniteCheckedException If failed. + */ + public UpdateResult processFast(Object[] args) throws IgniteCheckedException { + if (fastUpdate != null) + return fastUpdate.execute(cacheContext().cache(), args); + + return null; + } + + /** + * @return {@code True} if predefined rows exist. + */ + public boolean hasRows() { + return !F.isEmpty(rows); + } + + /** + * Extract rows from plan without performing any query. + * + * @param args Original query arguments. + * @return {@link List} of rows from the plan for a single query. + * For example, if we have multiple args in a query:
+ * {@code INSERT INTO person VALUES (k1, v1), (k2, v2), (k3, v3);}
+ * we will get a {@link List} of {@link List} with items {@code {[k1, v1], [k2, v2], [k3, v3]}}. + * @throws IgniteCheckedException if failed. + */ + public List> createRows(Object[] args) throws IgniteCheckedException { + assert rowsNum > 0 && !F.isEmpty(colNames); + + List> res = new ArrayList<>(rowsNum); + + GridQueryRowDescriptor desc = tbl.rowDescriptor(); + + extractArgsValues(args, res, desc); + + return res; + } + + /** + * Extract rows from plan without performing any query. + * + * @param argss Batch of arguments. + * @return {@link List} of rows from the plan for each query. + * For example, if we have a batch of queries with multiple args:
+ * + * INSERT INTO person VALUES (k1, v1), (k2, v2), (k3, v3);
+ * INSERT INTO person VALUES (k4, v4), (k5, v5), (k6, v6);
+ * + * we will get a {@link List} of {@link List} of {@link List} with items:
+ * + * {[k1, v1], [k2, v2], [k3, v3]},
+ * {[k4, v4], [k5, v5], [k6, v6]}
+ * + * @throws IgniteCheckedException If failed. + */ + public List>> createRows(List argss) throws IgniteCheckedException { + assert rowsNum > 0 && !F.isEmpty(colNames); + assert argss != null; + + List>> resPerQry = new ArrayList<>(argss.size()); + + GridQueryRowDescriptor desc = tbl.rowDescriptor(); + + for (Object[] args : argss) { + List> res = new ArrayList<>(); + + resPerQry.add(res); + + extractArgsValues(args, res, desc); + } + + return resPerQry; + } + + /** + * Extracts values from arguments. + * + * @param args Arguments. + * @param res Result list where to put values to. + * @param desc Row descriptor. + * @throws IgniteCheckedException If failed. + */ + private void extractArgsValues(Object[] args, List> res, GridQueryRowDescriptor desc) + throws IgniteCheckedException { + assert res != null; + + for (List row : rows) { + List resRow = new ArrayList<>(); + + for (int j = 0; j < colNames.length; j++) { + Object colVal; + if (fillAbsentPKsWithDefaults) + colVal = row.size() > j ? row.get(j).get(args) : null; + else + colVal = row.get(j).get(args); + + if (j == keyColIdx || j == valColIdx) { + Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); + + colVal = DmlUtils.convert(colVal, desc, colCls, colTypes[j], colNames[j]); + } + + resRow.add(colVal); + } + + res.add(resRow); + } + } + + /** + * @return Update mode. + */ + public UpdateMode mode() { + return mode; + } + + /** + * @return Cache context. + */ + public GridCacheContext cacheContext() { + return tbl.cacheContext(); + } + + /** + * @return Distributed plan info (for skip-reducer mode). + */ + @Nullable public DmlDistributedPlanInfo distributedPlan() { + return distributed; + } + + /** + * @return Row count. + */ + public int rowCount() { + return rowsNum; + } + + /** + * @return Select query. + */ + public String selectQuery() { + return selectQry; + } + + /** + * @return Local subquery flag. + */ + public boolean isLocalSubquery() { + return isLocSubqry; + } + + /** + * @return {@code true} is the SELECT query may be executed in lazy mode. + */ + public boolean canSelectBeLazy() { + return canSelectBeLazy; + } +} From 217784707b2ebeadd6e92e82eb51dbc4d92437e5 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 14:12:29 +1000 Subject: [PATCH 22/33] IGNITE-22557 Whitespaces refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 67eae9aadddf3..be95c8a6c152e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -80,7 +80,7 @@ public final class UpdatePlan { /** Number of rows in rows based MERGE or INSERT. */ private final int rowsNum; - + /** Whether absent PK parts should be filled with defaults or not. */ private boolean fillAbsentPKsWithDefaults; From 6bb9e06d0007f3dfaab38b0c1a78d328b78db7b7 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Wed, 25 Sep 2024 14:16:20 +1000 Subject: [PATCH 23/33] IGNITE-22557 Whitespaces refactoring --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index be95c8a6c152e..e68b71682e27b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -83,7 +83,7 @@ public final class UpdatePlan { /** Whether absent PK parts should be filled with defaults or not. */ private boolean fillAbsentPKsWithDefaults; - + /** Arguments for fast UPDATE or DELETE. */ private final FastUpdate fastUpdate; @@ -135,7 +135,7 @@ public UpdatePlan( this.rows = rows; this.rowsNum = rowsNum; this.fillAbsentPKsWithDefaults = fillAbsentPKsWithDefaults; - + assert mode != null; assert tbl != null; @@ -469,7 +469,7 @@ private void extractArgsValues(Object[] args, List> res, GridQueryRowDes colVal = row.size() > j ? row.get(j).get(args) : null; else colVal = row.get(j).get(args); - + if (j == keyColIdx || j == valColIdx) { Class colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass(); From 776339b30a22ff6804acc91699abd4caba6403af Mon Sep 17 00:00:00 2001 From: 21518201 Date: Sun, 29 Sep 2024 10:56:56 +1000 Subject: [PATCH 24/33] IGNITE-22557 SqlPlanHistoryConfigTest moved to ignite-spring, SqlPlanHistoryIntegrationTest refactored with assumeFalse(), SqlPlanHistoryTracker#setHistorySize edited --- modules/calcite/pom.xml | 6 - .../SqlPlanHistoryIntegrationTest.java | 225 +++++++++--------- .../testsuites/IntegrationTestSuite.java | 1 - .../query/running/SqlPlanHistoryTracker.java | 4 +- .../src/test/config/plan-history-conf.xml | 0 .../internal/SqlPlanHistoryConfigTest.java} | 6 +- .../testsuites/IgniteSpringTestSuite.java | 5 +- 7 files changed, 122 insertions(+), 125 deletions(-) rename modules/{calcite => spring}/src/test/config/plan-history-conf.xml (100%) rename modules/{calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java => spring/src/test/java/org/apache/ignite/internal/SqlPlanHistoryConfigTest.java} (86%) diff --git a/modules/calcite/pom.xml b/modules/calcite/pom.xml index d4315a4b26955..d80b33b8258d3 100644 --- a/modules/calcite/pom.xml +++ b/modules/calcite/pom.xml @@ -64,12 +64,6 @@ test - - ${project.groupId} - ignite-spring - test - - org.apache.calcite calcite-core diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 6816500511e84..88440a4da6f18 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.query.Query; import org.apache.ignite.cache.query.ScanQuery; @@ -42,17 +43,21 @@ import org.apache.ignite.configuration.SqlConfiguration; import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; import org.apache.ignite.internal.processors.query.running.SqlPlan; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AssumptionViolatedException; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.junit.Assume.assumeFalse; + /** Tests for SQL plan history (Calcite engine). */ @RunWith(Parameterized.class) public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { @@ -130,6 +135,9 @@ public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { /** TextQuery. */ private final TextQuery textQry = new TextQuery<>("String", "2"); + /** Flag for queries with map-reduce phases. */ + private boolean isReducePhase; + /** Client mode flag. */ private boolean isClient; @@ -164,10 +172,10 @@ public static Collection params() { ); return cfg.setCacheConfiguration( - configureCahce("A", Integer.class, String.class), - configureCahce("B", Integer.class, String.class), - configureCahce("pers", Integer.class, Person.class), - configureCahce("org", Integer.class, Organization.class) + configureCache("A", Integer.class, String.class), + configureCache("B", Integer.class, String.class), + configureCache("pers", Integer.class, Person.class), + configureCache("org", Integer.class, Organization.class) ); } @@ -182,7 +190,7 @@ protected QueryEngineConfigurationEx configureSqlEngine() { * @return Cache configuration. */ @SuppressWarnings("unchecked") - private CacheConfiguration configureCahce(String name, Class... idxTypes) { + private CacheConfiguration configureCache(String name, Class... idxTypes) { return new CacheConfiguration() .setName(name) .setIndexedTypes(idxTypes) @@ -273,38 +281,32 @@ protected void setClientMode(boolean isClient) { * Clears current SQL plan history. */ public void resetPlanHistory() { - queryNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); - - mapNode().context().query().runningQueryManager().resetPlanHistoryMetrics(); + for (Ignite ignite : G.allGrids()) + ((IgniteEx)ignite).context().query().runningQueryManager().resetPlanHistoryMetrics(); } /** Checks successful JDBC queries. */ @Test - public void testJdbcQuery() throws SQLException { - if (loc) - return; - + public void testJdbcQuery() throws Exception { for (int i = 0; i < 2; i++) { jdbcQuery(SQL); - checkSqlPlanHistory(getExpectedHistorySize()); + checkSqlPlanHistory(1); } } /** Checks failed JDBC queries. */ @Test - public void testJdbcQueryFailed() { - if (loc) - return; - + public void testJdbcQueryFailed() throws Exception { try { jdbcQuery(SQL_FAILED); } - catch (Exception ignore) { - //No-Op + catch (Exception e) { + if (e instanceof AssumptionViolatedException) + throw e; } - checkSqlPlanHistory(getExpectedHistorySize()); + checkSqlPlanHistory(1); } /** Checks successful SqlFieldsQuery. */ @@ -334,31 +336,43 @@ public void testSqlFieldsCrossCacheQueryFailed() { /** Checks successful SqlFieldsQuery with reduce phase. */ @Test public void testSqlFieldsQueryWithReducePhase() { - if (loc) - return; + assumeFalse("Only distributed queries have the map and reduce phases", loc); - cacheQuery(sqlFieldsQryWithReducePhase, "pers"); + isReducePhase = true; + + try { + cacheQuery(sqlFieldsQryWithReducePhase, "pers"); - checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 3 : 1); + checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 3 : 1); + } + finally { + isReducePhase = false; + } } /** Checks failed SqlFieldsQuery with reduce phase. */ @Test public void testSqlFieldsQueryWithReducePhaseFailed() { - if (loc) - return; + assumeFalse("Only distributed queries have the map and reduce phases", loc); - for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { - try { - cacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); - } - catch (Exception ignore) { - //No-Op - } + isReducePhase = true; + + try { + for (int i = 0; i < sqlFieldsQryWithReducePhaseFailed.length; i++) { + try { + cacheQuery(sqlFieldsQryWithReducePhaseFailed[i], "pers"); + } + catch (Exception ignore) { + //No-Op + } - checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? i + 1 : 0); + checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? i + 1 : 0); - resetPlanHistory(); + resetPlanHistory(); + } + } + finally { + isReducePhase = false; } } @@ -389,46 +403,46 @@ public void testTextQuery() { /** Checks DML commands executed via JDBC. */ @Test public void testJdbcDml() throws SQLException { - runJdbcDml(dmlCmds, true); + runDml(new IgniteBiTuple<>(DmlQueryType.JDBC, dmlCmds), true); } /** Checks DML commands with joins executed via JDBC. */ @Test public void testJdbcDmlWithJoins() throws SQLException { - runJdbcDml(dmlCmdsWithJoins, false); + runDml(new IgniteBiTuple<>(DmlQueryType.JDBC, dmlCmdsWithJoins), false); } /** Checks DML commands executed via SqlFieldsQuery. */ @Test - public void testSqlFieldsDml() { - runSqlFieldsQueryDml(dmlCmds, true); + public void testSqlFieldsDml() throws SQLException { + runDml(new IgniteBiTuple<>(DmlQueryType.SQL_FIELDS, dmlCmds), true); } /** Checks DML commands with joins executed via SqlFieldsQuery. */ @Test - public void testSqlFieldsDmlWithJoins() { - runSqlFieldsQueryDml(dmlCmdsWithJoins, false); + public void testSqlFieldsDmlWithJoins() throws SQLException { + runDml(new IgniteBiTuple<>(DmlQueryType.SQL_FIELDS, dmlCmdsWithJoins), false); } /** Checks that older plan entries are evicted when maximum history size is reached. */ @Test - public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { - if (loc || (!loc && isFullyFetched)) - return; + public void testPlanHistoryEviction() throws Exception { + assumeFalse("No SQL plans are written on the client node when using the H2 engine", + isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME); for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { try { - cacheQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), "A"); + executeQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), (q) -> cacheQuery(q, "A")); } - catch (Exception ignore) { - //No-Op + catch (Exception e) { + if (e instanceof AssumptionViolatedException) + throw e; } } GridTestUtils.waitForCondition(() -> getSqlPlanHistory(queryNode()).size() == PLAN_HISTORY_SIZE, 1000); - checkSqlPlanHistory((isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 0 : - PLAN_HISTORY_SIZE); + checkSqlPlanHistory(PLAN_HISTORY_SIZE); Set qrys = getSqlPlanHistory(queryNode()).stream().map(SqlPlan::query).collect(Collectors.toSet()); @@ -442,19 +456,20 @@ public void testPlanHistoryEviction() throws IgniteInterruptedCheckedException { */ @Test public void testEntryReplacement() throws InterruptedException { - if (loc || !isFullyFetched || (isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME)) - return; + assumeFalse("With the H2 engine, scan counts can be added to SQL plans for local queries ", + loc && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME); + + assumeFalse("No SQL plans are written on the client node when using the H2 engine", + isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME); long[] timeStamps = new long[2]; for (int i = 0; i < 2; i++) { executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); - Collection plans = getSqlPlanHistory(queryNode()); - - assertEquals(1, plans.size()); + checkSqlPlanHistory(1); - timeStamps[i] = plans.stream().findFirst().get().startTime(); + timeStamps[i] = getSqlPlanHistory(queryNode()).stream().findFirst().get().startTime(); Thread.sleep(10); } @@ -480,7 +495,7 @@ public void runSuccessfulQuery(Query qry) { for (int i = 0; i < 2; i++) { cacheQuery(q, "A"); - checkSqlPlanHistory(getExpectedHistorySize()); + checkSqlPlanHistory(1); } }); } @@ -497,7 +512,7 @@ public void runFailedQuery(Query qry) { //No-Op } - checkSqlPlanHistory(getExpectedHistorySize()); + checkSqlPlanHistory(1); }); } @@ -517,8 +532,7 @@ public void runQueryWithoutPlan(Query qry) { * @param task Task to execute. */ public void executeQuery(Query qry, Consumer task) { - if (isClient && loc) - return; + assumeFalse("Local queries can't be executed on client nodes", isClient && loc); qry.setLocal(loc); @@ -528,7 +542,9 @@ public void executeQuery(Query qry, Consumer task) { /** * @param qry Query. */ - private void jdbcQuery(String qry) throws SQLException { + private void jdbcQuery(String qry) throws Exception { + assumeFalse("There is no 'local query' parameter for JDBC queries", loc); + try ( Connection conn = GridTestUtils.connect(queryNode(), null); Statement stmt = conn.createStatement() @@ -561,61 +577,34 @@ public void cacheQuery(Query qry, String cacheName) { } /** - * @param cmds Set of DML commands. + * @param qrysInfo Information about the DML operations that need to be executed (type, list of commands). * @param isSimpleQry Simple query flag. */ - public void runJdbcDml(List cmds, boolean isSimpleQry) throws SQLException { - executeJdbcDml((stmt) -> { - for (String cmd : cmds) { - try { - stmt.execute(cmd); - } - catch (SQLException e) { - throw new RuntimeException(e); - } - } - }, isSimpleQry); - } + public void runDml(IgniteBiTuple> qrysInfo, boolean isSimpleQry) throws SQLException { + assumeFalse("Local queries can't be executed on client nodes", isClient && loc); - /** - * @param task Task to execute. - * @param isSimpleQry Simple query flag. - */ - public void executeJdbcDml(Consumer task, boolean isSimpleQry) throws SQLException { - if (loc || !isFullyFetched) - return; + assumeFalse("There is no lazy mode for DML operations", !isFullyFetched); - try ( - Connection conn = GridTestUtils.connect(queryNode(), null); - Statement stmt = conn.createStatement() - ) { - task.accept(stmt); - } + DmlQueryType type = qrysInfo.get1(); - checkSqlPlanHistoryDml(3, isSimpleQry); - } + assumeFalse("There is no 'local query' parameter for JDBC queries", loc && type == DmlQueryType.JDBC); - /** - * @param cmds Set of DML commands. - * @param isSimpleQry Simple query flag. - */ - public void runSqlFieldsQueryDml(List cmds, boolean isSimpleQry) { - executeSqlFieldsQueryDml( - cache -> cmds.forEach(cmd -> cache.query(new SqlFieldsQuery(cmd).setLocal(loc))), - isSimpleQry); - } + List cmds = qrysInfo.get2(); - /** - * @param task Task to execute. - * @param isSimpleQry Simple query flag. - */ - public void executeSqlFieldsQueryDml(Consumer> task, boolean isSimpleQry) { - if (isClient && loc || !isFullyFetched) - return; - - IgniteCache cache = queryNode().getOrCreateCache("A"); + if (type == DmlQueryType.JDBC) { + try ( + Connection conn = GridTestUtils.connect(queryNode(), null); + Statement stmt = conn.createStatement() + ) { + for (String cmd : cmds) + stmt.execute(cmd); + } + } + else if (type == DmlQueryType.SQL_FIELDS) { + IgniteCache cache = queryNode().getOrCreateCache("A"); - task.accept(cache); + cmds.forEach(c -> cache.query(new SqlFieldsQuery(c).setLocal(loc))); + } checkSqlPlanHistoryDml(3, isSimpleQry); } @@ -627,11 +616,6 @@ public Collection getSqlPlanHistory(IgniteEx node) { return new ArrayList<>(node.context().query().runningQueryManager().planHistoryTracker().sqlPlanHistory()); } - /** */ - public int getExpectedHistorySize() { - return (isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? 0 : 1; - } - /** * Prepares SQL plan history entries for futher checking. * @@ -642,6 +626,12 @@ public void checkSqlPlanHistory(int size) { assertNotNull(sqlPlans); + if (!isReducePhase && isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { + assertEquals(0, sqlPlans.size()); + + sqlPlans = getSqlPlanHistory(mapNode()); + } + checkMetrics(size, sqlPlans); } @@ -745,4 +735,13 @@ public static int fail() { throw new IgniteSQLException("SQL function fail for test purpuses"); } } + + /** */ + private enum DmlQueryType { + /** JDBC query. */ + JDBC, + + /** SqlFields query. */ + SQL_FIELDS + } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index 8c62894dfd0fe..c0326e51ebc75 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -60,7 +60,6 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SortAggregateIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlDiagnosticIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryClientIntegrationTest; -import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryConfigIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2ClientIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2IntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryIntegrationTest; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index b5ec9b7cd6a64..b2942d6c73608 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -24,7 +24,7 @@ /** Class that manages recording and storing SQL plans. */ public class SqlPlanHistoryTracker { /** SQL plan history. */ - private final GridBoundedConcurrentLinkedHashSet sqlPlanHistory; + private GridBoundedConcurrentLinkedHashSet sqlPlanHistory; /** SQL plan history size. */ private int historySize; @@ -70,5 +70,7 @@ public Set sqlPlanHistory() { */ public void setHistorySize(int historySize) { this.historySize = historySize; + + sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashSet<>(historySize) : null; } } diff --git a/modules/calcite/src/test/config/plan-history-conf.xml b/modules/spring/src/test/config/plan-history-conf.xml similarity index 100% rename from modules/calcite/src/test/config/plan-history-conf.xml rename to modules/spring/src/test/config/plan-history-conf.xml diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/SqlPlanHistoryConfigTest.java similarity index 86% rename from modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java rename to modules/spring/src/test/java/org/apache/ignite/internal/SqlPlanHistoryConfigTest.java index 28213ed444b59..f5d3ff1ea316d 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryConfigIntegrationTest.java +++ b/modules/spring/src/test/java/org/apache/ignite/internal/SqlPlanHistoryConfigTest.java @@ -15,21 +15,21 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite.integration; +package org.apache.ignite.internal; import org.apache.ignite.Ignite; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; /** Test for Sql plan history configuration. */ -public class SqlPlanHistoryConfigIntegrationTest extends GridCommonAbstractTest { +public class SqlPlanHistoryConfigTest extends GridCommonAbstractTest { /** Sql plan history size in the XML Spring config. */ private static final int SQL_PLAN_HISTORY_SIZE_XML_CONFIG = 10; /** Checks that plan history size specified in XML config is respected. */ @Test public void testXmlConfigSqlPlanHistorySize() throws Exception { - String cfgPath = "modules/calcite/src/test/config/plan-history-conf.xml"; + String cfgPath = "modules/spring/src/test/config/plan-history-conf.xml"; Ignite ignite = startGridsWithSpringCtx(2, false, cfgPath); diff --git a/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java b/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java index bc05aa6ca87c7..50ac65e09125f 100644 --- a/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java +++ b/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.IgniteClientSpringBeanTest; import org.apache.ignite.internal.IgniteDynamicCacheConfigTest; import org.apache.ignite.internal.IgniteSpringBeanTest; +import org.apache.ignite.internal.SqlPlanHistoryConfigTest; import org.apache.ignite.internal.metric.RegexpMetricFilterTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtMultiBackupTest; import org.apache.ignite.internal.processors.resource.GridTransformSpringInjectionSelfTest; @@ -85,7 +86,9 @@ RegexpMetricFilterTest.class, // CDC tests. - CdcConfigurationTest.class + CdcConfigurationTest.class, + + SqlPlanHistoryConfigTest.class }) public class IgniteSpringTestSuite { } From 5fa16ac4cc6b7fd626c3cd0d8e5953cdf0ccdfd0 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Sun, 29 Sep 2024 12:05:00 +1000 Subject: [PATCH 25/33] IGNITE-22557 GridBoundedConcurrentLinkedHashSet has been changed back to GridBoundedConcurrentLinkedHashMap --- .../SqlPlanHistoryIntegrationTest.java | 44 ++++++++++++------- .../query/running/RunningQueryManager.java | 2 +- .../processors/query/running/SqlPlan.java | 10 ----- .../query/running/SqlPlanHistoryTracker.java | 22 +++++----- .../systemview/view/SqlPlanHistoryView.java | 17 +++---- 5 files changed, 47 insertions(+), 48 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 88440a4da6f18..4db4937bca324 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -24,7 +24,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -444,7 +446,9 @@ public void testPlanHistoryEviction() throws Exception { checkSqlPlanHistory(PLAN_HISTORY_SIZE); - Set qrys = getSqlPlanHistory(queryNode()).stream().map(SqlPlan::query).collect(Collectors.toSet()); + Set qrys = getSqlPlanHistory(queryNode()).keySet().stream() + .map(SqlPlan::query) + .collect(Collectors.toSet()); for (int i = 1; i <= PLAN_HISTORY_EXCESS; i++) assertFalse(qrys.contains(SQL + " where A.fail=" + i)); @@ -469,7 +473,7 @@ public void testEntryReplacement() throws InterruptedException { checkSqlPlanHistory(1); - timeStamps[i] = getSqlPlanHistory(queryNode()).stream().findFirst().get().startTime(); + timeStamps[i] = getSqlPlanHistory(queryNode()).values().stream().findFirst().get(); Thread.sleep(10); } @@ -612,8 +616,8 @@ else if (type == DmlQueryType.SQL_FIELDS) { /** * @param node Ignite node to check SQL plan history for. */ - public Collection getSqlPlanHistory(IgniteEx node) { - return new ArrayList<>(node.context().query().runningQueryManager().planHistoryTracker().sqlPlanHistory()); + public Map getSqlPlanHistory(IgniteEx node) { + return node.context().query().runningQueryManager().planHistoryTracker().sqlPlanHistory(); } /** @@ -622,7 +626,7 @@ public Collection getSqlPlanHistory(IgniteEx node) { * @param size Number of SQL plan entries expected to be in the history. */ public void checkSqlPlanHistory(int size) { - Collection sqlPlans = getSqlPlanHistory(queryNode()); + Map sqlPlans = getSqlPlanHistory(queryNode()); assertNotNull(sqlPlans); @@ -642,7 +646,7 @@ public void checkSqlPlanHistory(int size) { * @param isSimpleQry Simple query flag. */ public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { - Collection sqlPlans = getSqlPlanHistory(queryNode()); + Map sqlPlans = getSqlPlanHistory(queryNode()); assertNotNull(sqlPlans); @@ -654,7 +658,9 @@ public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { else check = "the following " + (loc ? "local " : "") + "query has been executed:"; - sqlPlans = sqlPlans.stream().filter(p -> p.plan().contains(check)).collect(Collectors.toList()); + sqlPlans = sqlPlans.entrySet().stream() + .filter(e -> e.getKey().plan().contains(check)) + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); checkMetrics((!isSimpleQry && !loc) ? size : 0, getSqlPlanHistory(mapNode())); } @@ -668,10 +674,14 @@ public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { * @param size Number of SQL plan entries expected to be in the history. * @param sqlPlans Sql plans recorded in the history. */ - public void checkMetrics(int size, Collection sqlPlans) { + public void checkMetrics(int size, Map sqlPlans) { if (size == 1 && sqlPlans.size() == 2) { - String plan1 = new ArrayList<>(sqlPlans).get(0).plan(); - String plan2 = new ArrayList<>(sqlPlans).get(1).plan(); + List> sortedPlans = new ArrayList<>(sqlPlans.entrySet()).stream() + .sorted(Comparator.comparing(Map.Entry::getValue)) + .collect(Collectors.toList()); + + String plan1 = sortedPlans.get(0).getKey().plan(); + String plan2 = sortedPlans.get(1).getKey().plan(); assertTrue(plan2.contains(plan1) && plan2.contains("/* scanCount")); } @@ -681,15 +691,15 @@ public void checkMetrics(int size, Collection sqlPlans) { if (size == 0) return; - for (SqlPlan plan : sqlPlans) { - assertEquals(loc, plan.local()); - assertEquals(sqlEngine.toString(), plan.engine()); + for (Map.Entry plan : sqlPlans.entrySet()) { + assertEquals(loc, plan.getKey().local()); + assertEquals(sqlEngine, plan.getKey().engine()); - assertNotNull(plan.plan()); - assertNotNull(plan.query()); - assertNotNull(plan.schema()); + assertNotNull(plan.getKey().plan()); + assertNotNull(plan.getKey().query()); + assertNotNull(plan.getKey().schema()); - assertTrue(plan.startTime() > 0); + assertTrue(plan.getValue() > 0); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java index 96a216c6da90e..bdcb6d4a194a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/RunningQueryManager.java @@ -215,7 +215,7 @@ public RunningQueryManager(GridKernalContext ctx) { ctx.systemView().registerView(SQL_PLAN_HIST_VIEW, SQL_PLAN_HIST_VIEW_DESC, new SqlPlanHistoryViewWalker(), - new ArrayList<>(planHistTracker.sqlPlanHistory()), + planHistTracker.sqlPlanHistory().entrySet(), SqlPlanHistoryView::new); MetricRegistryImpl userMetrics = ctx.metric().registry(SQL_USER_QUERIES_REG_NAME); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java index bac6fdd129c23..b77d67e919507 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java @@ -38,9 +38,6 @@ public class SqlPlan { /** SQL engine. */ private final String engine; - /** Query start timestamp. */ - private final long startTime; - /** Pre-calculated hash code. */ private final int hash; @@ -63,8 +60,6 @@ public SqlPlan( this.loc = loc; this.engine = engine; - startTime = U.currentTimeMillis(); - hash = Objects.hash(plan, qry, schema, loc, engine); } @@ -93,11 +88,6 @@ public String engine() { return engine; } - /** */ - public long startTime() { - return startTime; - } - /** {@inheritDoc} */ @Override public int hashCode() { return hash; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java index b2942d6c73608..756a0fc73b914 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlanHistoryTracker.java @@ -18,13 +18,14 @@ package org.apache.ignite.internal.processors.query.running; import java.util.Collections; -import java.util.Set; -import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet; +import java.util.Map; +import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; +import org.apache.ignite.internal.util.typedef.internal.U; /** Class that manages recording and storing SQL plans. */ public class SqlPlanHistoryTracker { /** SQL plan history. */ - private GridBoundedConcurrentLinkedHashSet sqlPlanHistory; + private GridBoundedConcurrentLinkedHashMap sqlPlanHistory; /** SQL plan history size. */ private int historySize; @@ -35,7 +36,7 @@ public class SqlPlanHistoryTracker { public SqlPlanHistoryTracker(int historySize) { this.historySize = historySize; - sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashSet<>(historySize) : null; + sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashMap<>(historySize) : null; } /** @@ -51,18 +52,15 @@ public void addPlan(String plan, String qry, String schema, boolean loc, String SqlPlan sqlPlan = new SqlPlan(plan, qry, schema, loc, engine); - if (sqlPlanHistory.contains(sqlPlan)) - sqlPlanHistory.remove(sqlPlan); - - sqlPlanHistory.add(sqlPlan); + sqlPlanHistory.put(sqlPlan, U.currentTimeMillis()); } /** */ - public Set sqlPlanHistory() { + public Map sqlPlanHistory() { if (historySize <= 0) - return Collections.emptySet(); + return Collections.emptyMap(); - return Collections.unmodifiableSet(sqlPlanHistory); + return Collections.unmodifiableMap(sqlPlanHistory); } /** @@ -71,6 +69,6 @@ public Set sqlPlanHistory() { public void setHistorySize(int historySize) { this.historySize = historySize; - sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashSet<>(historySize) : null; + sqlPlanHistory = (historySize > 0) ? new GridBoundedConcurrentLinkedHashMap<>(historySize) : null; } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java index 0ef46044357d4..1666fbb8679cc 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/SqlPlanHistoryView.java @@ -18,18 +18,19 @@ package org.apache.ignite.spi.systemview.view; import java.util.Date; +import java.util.Map; import org.apache.ignite.internal.managers.systemview.walker.Order; import org.apache.ignite.internal.processors.query.running.SqlPlan; /** */ public class SqlPlanHistoryView { /** SQL plan. */ - private final SqlPlan plan; + private final Map.Entry plan; /** * @param plan SQL plan. */ - public SqlPlanHistoryView(SqlPlan plan) { + public SqlPlanHistoryView(Map.Entry plan) { this.plan = plan; } @@ -38,7 +39,7 @@ public SqlPlanHistoryView(SqlPlan plan) { */ @Order public String schemaName() { - return plan.schema(); + return plan.getKey().schema(); } /** @@ -46,7 +47,7 @@ public String schemaName() { */ @Order(1) public String sql() { - return plan.query(); + return plan.getKey().query(); } /** @@ -54,7 +55,7 @@ public String sql() { */ @Order(2) public String plan() { - return plan.plan(); + return plan.getKey().plan(); } /** @@ -62,7 +63,7 @@ public String plan() { */ @Order(3) public boolean local() { - return plan.local(); + return plan.getKey().local(); } /** @@ -70,7 +71,7 @@ public boolean local() { */ @Order(4) public String engine() { - return plan.engine(); + return plan.getKey().engine(); } /** @@ -78,6 +79,6 @@ public String engine() { */ @Order(5) public Date lastStartTime() { - return new Date(plan.startTime()); + return new Date(plan.getValue()); } } From e6eeaebf72154671312adddbee54fb98993f67bc Mon Sep 17 00:00:00 2001 From: 21518201 Date: Sun, 29 Sep 2024 13:50:40 +1000 Subject: [PATCH 26/33] IGNITE-22557 sqlEngine and isClient added as parameters to SqlPlanHistoryIntegrationTest --- .../SqlPlanHistoryIntegrationTest.java | 62 ++++++++++++------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 4db4937bca324..7cc09d2d38a54 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -27,6 +27,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -140,25 +141,42 @@ public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { /** Flag for queries with map-reduce phases. */ private boolean isReducePhase; - /** Client mode flag. */ - private boolean isClient; - /** SQL engine. */ - protected String sqlEngine = CalciteQueryEngineConfiguration.ENGINE_NAME; + @Parameterized.Parameter + public String sqlEngine; + + /** Client mode flag. */ + @Parameterized.Parameter(1) + public boolean isClient; /** Local query flag. */ - @Parameterized.Parameter + @Parameterized.Parameter(2) public boolean loc; /** Fully-fetched query flag. */ - @Parameterized.Parameter(1) + @Parameterized.Parameter(3) public boolean isFullyFetched; /** */ - @Parameterized.Parameters(name = "loc={0}, fullyFetched={1}") + @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1} loc={2}, fullyFetched={3}") public static Collection params() { return Arrays.asList(new Object[][] { - {true, true}, {true, false}, {false, true}, {false, false} + {CalciteQueryEngineConfiguration.ENGINE_NAME, true, true, true}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, true, true, false}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, true, false, true}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, true, false, false}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, false, true, true}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, false, true, false}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, false, false, true}, + {CalciteQueryEngineConfiguration.ENGINE_NAME, false, false, false}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, true, true, true}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, true, true, false}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, true, false, true}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, true, false, false}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, false, true, true}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, false, true, false}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, false, false, true}, + {IndexingQueryEngineConfiguration.ENGINE_NAME, false, false, false} }); } @@ -183,7 +201,10 @@ public static Collection params() { /** */ protected QueryEngineConfigurationEx configureSqlEngine() { - return new CalciteQueryEngineConfiguration(); + if (sqlEngine.equals(CalciteQueryEngineConfiguration.ENGINE_NAME)) + return new CalciteQueryEngineConfiguration(); + else + return new IndexingQueryEngineConfiguration(); } /** @@ -203,9 +224,12 @@ private CacheConfiguration configureCache(String name, Class... idxTypes) { * @return Ignite node where queries are executed. */ protected IgniteEx queryNode() { - IgniteEx node = grid(0); + IgniteEx node = isClient ? grid(2) : grid(0); - assertFalse(node.context().clientNode()); + if (isClient) + assertTrue(node.context().clientNode()); + else + assertFalse(node.context().clientNode()); return node; } @@ -228,12 +252,13 @@ protected IgniteEx mapNode() { */ protected void startTestGrid() throws Exception { startGrids(2); + + if (isClient) + startClientGrid(2); } /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - + @Override protected void beforeTest() throws Exception { startTestGrid(); IgniteCache cacheA = queryNode().cache("A"); @@ -254,17 +279,10 @@ protected void startTestGrid() throws Exception { } /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { - super.afterTestsStopped(); - + @Override protected void afterTest() throws Exception { stopAllGrids(); } - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - resetPlanHistory(); - } - /** * @param sqlEngine Sql engine. */ From 9a68a1059f733b8cef11be823b88e2d05f04c2e7 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Sun, 29 Sep 2024 17:09:42 +1000 Subject: [PATCH 27/33] IGNITE-22557 other options for SqlPlanHistoryIntegrationTest#params --- .../SqlPlanHistoryIntegrationTest.java | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 7cc09d2d38a54..c48d275f2022e 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -27,7 +27,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -180,6 +179,43 @@ public static Collection params() { }); } +// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1} loc={2}, fullyFetched={3}") +// public static Collection params() { +// return Arrays.stream(new Object[][]{ +// {CalciteQueryEngineConfiguration.ENGINE_NAME}, +// {IndexingQueryEngineConfiguration.ENGINE_NAME} +// }).flatMap(arr -> Arrays.stream(new Boolean[]{true, false}) +// .flatMap(isClient -> Arrays.stream(new Boolean[]{true, false}) +// .flatMap(loc -> Arrays.stream(new Boolean[]{true, false}) +// .map(fullyFetched -> new Object[]{arr[0], isClient, loc, fullyFetched})))) +// .collect(Collectors.toList()); +// } +// +// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1}, loc={2}, fullyFetched={3}") +// public static Collection params() { +// return Arrays.asList(Stream.of( +// CalciteQueryEngineConfiguration.ENGINE_NAME, IndexingQueryEngineConfiguration.ENGINE_NAME +// ).flatMap(engineName -> Stream.of(true, false).flatMap(isClient -> Stream.of( +// true, false +// ).flatMap(loc -> Stream.of( +// true, false +// ).flatMap(fullyFetched -> Stream.of( +// new Object[]{engineName, isClient, loc, fullyFetched} +// ) +// ))) +// .toArray(Object[][]::new)); +// } +// +// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1}, loc={2}, fullyFetched={3}") +// public static Collection params() { +// return Arrays.asList(Stream.of(CalciteQueryEngineConfiguration.ENGINE_NAME, IndexingQueryEngineConfiguration.ENGINE_NAME) +// .flatMap(engineName -> Stream.of(true, false).flatMap(isClient -> Stream.of(true, false) +// .flatMap(loc -> Stream.of(true, false) +// .flatMap(fullyFetched -> Stream.of(new Object[] { engineName, isClient, loc, fullyFetched }) +// .iterator())))) +// .toArray(Object[][]::new)); +// } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); From 31faf0c13ae217828cd8466a828579de2d3f6045 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Thu, 3 Oct 2024 10:37:40 +1000 Subject: [PATCH 28/33] IGNITE-22557 SqlPlanHistoryIntegrationTest refactored, SqlPlanHistoryClientIntegrationTest, SqlPlanHistoryH2IntegrationTest and SqlPlanHistoryH2ClientIntegrationTest deleted --- .../SqlPlanHistoryClientIntegrationTest.java | 47 ------------- ...SqlPlanHistoryH2ClientIntegrationTest.java | 47 ------------- .../SqlPlanHistoryH2IntegrationTest.java | 36 ---------- .../SqlPlanHistoryIntegrationTest.java | 70 ++++--------------- .../testsuites/IntegrationTestSuite.java | 6 -- 5 files changed, 14 insertions(+), 192 deletions(-) delete mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java delete mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java delete mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java deleted file mode 100644 index 2c7cbbe574801..0000000000000 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryClientIntegrationTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.calcite.integration; - -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.junits.JUnitAssertAware; - -/** Tests for SQL plan history from client (Calcite engine). */ -public class SqlPlanHistoryClientIntegrationTest extends SqlPlanHistoryIntegrationTest { - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - setClientMode(true); - } - - /** {@inheritDoc} */ - @Override protected IgniteEx queryNode() { - IgniteEx node = grid(2); - - JUnitAssertAware.assertTrue(node.context().clientNode()); - - return node; - } - - /** {@inheritDoc} */ - @Override protected void startTestGrid() throws Exception { - startGrids(2); - - startClientGrid(2); - } -} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java deleted file mode 100644 index 51a1aedc2ed17..0000000000000 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2ClientIntegrationTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.calcite.integration; - -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.junits.JUnitAssertAware; - -/** Tests for SQL plan history from client (H2 engine). */ -public class SqlPlanHistoryH2ClientIntegrationTest extends SqlPlanHistoryH2IntegrationTest { - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - setClientMode(true); - } - - /** {@inheritDoc} */ - @Override protected IgniteEx queryNode() { - IgniteEx node = grid(2); - - JUnitAssertAware.assertTrue(node.context().clientNode()); - - return node; - } - - /** {@inheritDoc} */ - @Override protected void startTestGrid() throws Exception { - startGrids(2); - - startClientGrid(2); - } -} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java deleted file mode 100644 index 3835e24738467..0000000000000 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryH2IntegrationTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.calcite.integration; - -import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; -import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; - -/** Tests for SQL plan history (H2 engine). */ -public class SqlPlanHistoryH2IntegrationTest extends SqlPlanHistoryIntegrationTest { - /** {@inheritDoc} */ - @Override protected QueryEngineConfigurationEx configureSqlEngine() { - return new IndexingQueryEngineConfiguration(); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - setSqlEngine(IndexingQueryEngineConfiguration.ENGINE_NAME); - } -} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index c48d275f2022e..56b5d7dd8cd52 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -157,65 +157,18 @@ public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { public boolean isFullyFetched; /** */ - @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1} loc={2}, fullyFetched={3}") + @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1} loc={2}, isFullyFetched={3}") public static Collection params() { - return Arrays.asList(new Object[][] { - {CalciteQueryEngineConfiguration.ENGINE_NAME, true, true, true}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, true, true, false}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, true, false, true}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, true, false, false}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, false, true, true}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, false, true, false}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, false, false, true}, - {CalciteQueryEngineConfiguration.ENGINE_NAME, false, false, false}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, true, true, true}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, true, true, false}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, true, false, true}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, true, false, false}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, false, true, true}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, false, true, false}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, false, false, true}, - {IndexingQueryEngineConfiguration.ENGINE_NAME, false, false, false} - }); + return Arrays.stream(new Object[][]{ + {CalciteQueryEngineConfiguration.ENGINE_NAME}, + {IndexingQueryEngineConfiguration.ENGINE_NAME} + }).flatMap(sqlEngine -> Arrays.stream(new Boolean[]{true, false}) + .flatMap(isClient -> Arrays.stream(new Boolean[]{true, false}) + .flatMap(loc -> Arrays.stream(new Boolean[]{true, false}) + .map(isFullyFetched -> new Object[]{sqlEngine[0], isClient, loc, isFullyFetched}))) + ).collect(Collectors.toList()); } -// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1} loc={2}, fullyFetched={3}") -// public static Collection params() { -// return Arrays.stream(new Object[][]{ -// {CalciteQueryEngineConfiguration.ENGINE_NAME}, -// {IndexingQueryEngineConfiguration.ENGINE_NAME} -// }).flatMap(arr -> Arrays.stream(new Boolean[]{true, false}) -// .flatMap(isClient -> Arrays.stream(new Boolean[]{true, false}) -// .flatMap(loc -> Arrays.stream(new Boolean[]{true, false}) -// .map(fullyFetched -> new Object[]{arr[0], isClient, loc, fullyFetched})))) -// .collect(Collectors.toList()); -// } -// -// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1}, loc={2}, fullyFetched={3}") -// public static Collection params() { -// return Arrays.asList(Stream.of( -// CalciteQueryEngineConfiguration.ENGINE_NAME, IndexingQueryEngineConfiguration.ENGINE_NAME -// ).flatMap(engineName -> Stream.of(true, false).flatMap(isClient -> Stream.of( -// true, false -// ).flatMap(loc -> Stream.of( -// true, false -// ).flatMap(fullyFetched -> Stream.of( -// new Object[]{engineName, isClient, loc, fullyFetched} -// ) -// ))) -// .toArray(Object[][]::new)); -// } -// -// @Parameterized.Parameters(name = "sqlEngine={0}, isClient={1}, loc={2}, fullyFetched={3}") -// public static Collection params() { -// return Arrays.asList(Stream.of(CalciteQueryEngineConfiguration.ENGINE_NAME, IndexingQueryEngineConfiguration.ENGINE_NAME) -// .flatMap(engineName -> Stream.of(true, false).flatMap(isClient -> Stream.of(true, false) -// .flatMap(loc -> Stream.of(true, false) -// .flatMap(fullyFetched -> Stream.of(new Object[] { engineName, isClient, loc, fullyFetched }) -// .iterator())))) -// .toArray(Object[][]::new)); -// } - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -295,6 +248,9 @@ protected void startTestGrid() throws Exception { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { + if (isClient && loc) + return; + startTestGrid(); IgniteCache cacheA = queryNode().cache("A"); @@ -538,6 +494,8 @@ public void testEntryReplacement() throws InterruptedException { /** Checks that SQL plan history remains empty if history size is set to zero. */ @Test public void testEmptyPlanHistory() { + assumeFalse("Local queries can't be executed on client nodes", isClient && loc); + queryNode().context().query().runningQueryManager().planHistoryTracker().setHistorySize(0); executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index c0326e51ebc75..1514c2f203c63 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -59,9 +59,6 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SetOpIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SortAggregateIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlDiagnosticIntegrationTest; -import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryClientIntegrationTest; -import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2ClientIntegrationTest; -import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryH2IntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.SqlPlanHistoryIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.StatisticsCommandDdlIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.StdSqlOperatorsTest; @@ -145,9 +142,6 @@ ViewsIntegrationTest.class, SqlPlanHistoryConfigIntegrationTest.class, SqlPlanHistoryIntegrationTest.class, - SqlPlanHistoryClientIntegrationTest.class, - SqlPlanHistoryH2IntegrationTest.class, - SqlPlanHistoryH2ClientIntegrationTest.class, }) public class IntegrationTestSuite { } From 2e6fdbe6aaa47f800a6de8015e8e793ba615337b Mon Sep 17 00:00:00 2001 From: 21518201 Date: Thu, 3 Oct 2024 10:42:40 +1000 Subject: [PATCH 29/33] IGNITE-22557 Javadoc for SqlPlanHistoryIntegrationTest corrected --- .../calcite/integration/SqlPlanHistoryIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 56b5d7dd8cd52..a4c12cd3596a5 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -60,7 +60,7 @@ import static org.junit.Assume.assumeFalse; -/** Tests for SQL plan history (Calcite engine). */ +/** Tests for SQL plan history. */ @RunWith(Parameterized.class) public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { /** SQL plan history size. */ From 850f43c29259e856ef173eef01ff1176b3881e7e Mon Sep 17 00:00:00 2001 From: 21518201 Date: Mon, 7 Oct 2024 17:09:50 +1000 Subject: [PATCH 30/33] IGNITE-22557 SqlPlanHistoryIntegrationTest refactored --- .../SqlPlanHistoryIntegrationTest.java | 247 ++++++++---------- 1 file changed, 113 insertions(+), 134 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index a4c12cd3596a5..2c77a91ea2c80 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -45,11 +46,13 @@ import org.apache.ignite.configuration.SqlConfiguration; import org.apache.ignite.indexing.IndexingQueryEngineConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryEngineConfigurationEx; import org.apache.ignite.internal.processors.query.running.SqlPlan; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -87,20 +90,24 @@ public class SqlPlanHistoryIntegrationTest extends GridCommonAbstractTest { " union select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o" + " where p.orgId=o._key and o._key=102"; - /** Set of simple DML commands. */ - private final List dmlCmds = Arrays.asList( - "insert into A.String (_key, _val) values(101, '101')", - "update A.String set _val='111' where _key=101", - "delete from A.String where _key=101" + /** List of simple DML commands and the simple queries flag. */ + private final IgniteBiTuple, Boolean> dmlCmds = new IgniteBiTuple<>( + Arrays.asList( + "insert into A.String (_key, _val) values(101, '101')", + "update A.String set _val='111' where _key=101", + "delete from A.String where _key=101" + ), true ); - /** Set of DML commands with joins. */ - private final List dmlCmdsWithJoins = Arrays.asList( - "insert into A.String (_key, _val) select o._key, p.name " + - "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", - "update A.String set _val = 'updated' where _key in " + - "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", - "delete from A.String where _key in (select orgId from \"pers\".Person)" + /** List of DML commands with joins and the simple queries flag. */ + private final IgniteBiTuple, Boolean> dmlCmdsWithJoins = new IgniteBiTuple<>( + Arrays.asList( + "insert into A.String (_key, _val) select o._key, p.name " + + "from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key", + "update A.String set _val = 'updated' where _key in " + + "(select o._key from \"pers\".Person p, \"org\".Organization o where p.orgId=o._key)", + "delete from A.String where _key in (select orgId from \"pers\".Person)" + ), false ); /** Successful SqlFieldsQuery. */ @@ -248,8 +255,7 @@ protected void startTestGrid() throws Exception { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - if (isClient && loc) - return; + assumeFalse("Local queries can't be executed on client nodes", isClient && loc); startTestGrid(); @@ -275,28 +281,6 @@ protected void startTestGrid() throws Exception { stopAllGrids(); } - /** - * @param sqlEngine Sql engine. - */ - protected void setSqlEngine(String sqlEngine) { - this.sqlEngine = sqlEngine; - } - - /** - * @param isClient Client more flag. - */ - protected void setClientMode(boolean isClient) { - this.isClient = isClient; - } - - /** - * Clears current SQL plan history. - */ - public void resetPlanHistory() { - for (Ignite ignite : G.allGrids()) - ((IgniteEx)ignite).context().query().runningQueryManager().resetPlanHistoryMetrics(); - } - /** Checks successful JDBC queries. */ @Test public void testJdbcQuery() throws Exception { @@ -380,7 +364,8 @@ public void testSqlFieldsQueryWithReducePhaseFailed() { checkSqlPlanHistory((!isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) ? i + 1 : 0); - resetPlanHistory(); + for (Ignite ignite : G.allGrids()) + ((IgniteEx)ignite).context().query().runningQueryManager().resetPlanHistoryMetrics(); } } finally { @@ -414,26 +399,26 @@ public void testTextQuery() { /** Checks DML commands executed via JDBC. */ @Test - public void testJdbcDml() throws SQLException { - runDml(new IgniteBiTuple<>(DmlQueryType.JDBC, dmlCmds), true); + public void testJdbcDml() { + runJdbcDml(dmlCmds); } /** Checks DML commands with joins executed via JDBC. */ @Test - public void testJdbcDmlWithJoins() throws SQLException { - runDml(new IgniteBiTuple<>(DmlQueryType.JDBC, dmlCmdsWithJoins), false); + public void testJdbcDmlWithJoins() { + runJdbcDml(dmlCmdsWithJoins); } /** Checks DML commands executed via SqlFieldsQuery. */ @Test - public void testSqlFieldsDml() throws SQLException { - runDml(new IgniteBiTuple<>(DmlQueryType.SQL_FIELDS, dmlCmds), true); + public void testSqlFieldsDml() { + runSqlFieldsDml(dmlCmds); } /** Checks DML commands with joins executed via SqlFieldsQuery. */ @Test - public void testSqlFieldsDmlWithJoins() throws SQLException { - runDml(new IgniteBiTuple<>(DmlQueryType.SQL_FIELDS, dmlCmdsWithJoins), false); + public void testSqlFieldsDmlWithJoins() { + runSqlFieldsDml(dmlCmdsWithJoins); } /** Checks that older plan entries are evicted when maximum history size is reached. */ @@ -444,11 +429,12 @@ public void testPlanHistoryEviction() throws Exception { for (int i = 1; i <= (PLAN_HISTORY_SIZE + PLAN_HISTORY_EXCESS); i++) { try { - executeQuery(new SqlFieldsQuery(SQL + " where A.fail()=" + i), (q) -> cacheQuery(q, "A")); + SqlFieldsQuery qry = new SqlFieldsQuery(SQL + " where A.fail()=" + i); + + cacheQuery(qry.setLocal(loc), "A"); } - catch (Exception e) { - if (e instanceof AssumptionViolatedException) - throw e; + catch (Exception ignore) { + //No-op } } @@ -469,7 +455,7 @@ public void testPlanHistoryEviction() throws Exception { * the beginning time). */ @Test - public void testEntryReplacement() throws InterruptedException { + public void testEntryReplacement() throws IgniteInterruptedCheckedException { assumeFalse("With the H2 engine, scan counts can be added to SQL plans for local queries ", loc && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME); @@ -479,13 +465,13 @@ public void testEntryReplacement() throws InterruptedException { long[] timeStamps = new long[2]; for (int i = 0; i < 2; i++) { - executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); + cacheQuery(sqlFieldsQry.setLocal(loc), "A"); checkSqlPlanHistory(1); timeStamps[i] = getSqlPlanHistory(queryNode()).values().stream().findFirst().get(); - Thread.sleep(10); + U.sleep(10); } assertTrue(timeStamps[1] > timeStamps[0]); @@ -494,11 +480,9 @@ public void testEntryReplacement() throws InterruptedException { /** Checks that SQL plan history remains empty if history size is set to zero. */ @Test public void testEmptyPlanHistory() { - assumeFalse("Local queries can't be executed on client nodes", isClient && loc); - queryNode().context().query().runningQueryManager().planHistoryTracker().setHistorySize(0); - executeQuery(sqlFieldsQry, (q) -> cacheQuery(q, "A")); + cacheQuery(sqlFieldsQry.setLocal(loc), "A"); assertTrue(getSqlPlanHistory(queryNode()).isEmpty()); } @@ -507,52 +491,34 @@ public void testEmptyPlanHistory() { * @param qry Query. */ public void runSuccessfulQuery(Query qry) { - executeQuery(qry, (q) -> { - for (int i = 0; i < 2; i++) { - cacheQuery(q, "A"); + for (int i = 0; i < 2; i++) { + cacheQuery(qry.setLocal(loc), "A"); - checkSqlPlanHistory(1); - } - }); + checkSqlPlanHistory(1); + } } /** * @param qry Query. */ public void runFailedQuery(Query qry) { - executeQuery(qry, (q) -> { - try { - cacheQuery(q, "A"); - } - catch (Exception ignore) { - //No-Op - } + try { + cacheQuery(qry.setLocal(loc), "A"); + } + catch (Exception ignore) { + //No-Op + } - checkSqlPlanHistory(1); - }); + checkSqlPlanHistory(1); } /** * @param qry Query. */ public void runQueryWithoutPlan(Query qry) { - executeQuery(qry, (q) -> { - cacheQuery(q, "A"); + cacheQuery(qry.setLocal(loc), "A"); - checkSqlPlanHistory(0); - }); - } - - /** - * @param qry Query. - * @param task Task to execute. - */ - public void executeQuery(Query qry, Consumer task) { - assumeFalse("Local queries can't be executed on client nodes", isClient && loc); - - qry.setLocal(loc); - - task.accept(qry); + checkSqlPlanHistory(0); } /** @@ -593,21 +559,12 @@ public void cacheQuery(Query qry, String cacheName) { } /** - * @param qrysInfo Information about the DML operations that need to be executed (type, list of commands). - * @param isSimpleQry Simple query flag. + * @param qrysInfo DML commands info (queries, simple query flag). */ - public void runDml(IgniteBiTuple> qrysInfo, boolean isSimpleQry) throws SQLException { - assumeFalse("Local queries can't be executed on client nodes", isClient && loc); - - assumeFalse("There is no lazy mode for DML operations", !isFullyFetched); - - DmlQueryType type = qrysInfo.get1(); - - assumeFalse("There is no 'local query' parameter for JDBC queries", loc && type == DmlQueryType.JDBC); - - List cmds = qrysInfo.get2(); + public void runJdbcDml(IgniteBiTuple, Boolean> qrysInfo) { + assumeFalse("There is no 'local query' parameter for JDBC queries", loc); - if (type == DmlQueryType.JDBC) { + executeDml(qrysInfo, (cmds) -> { try ( Connection conn = GridTestUtils.connect(queryNode(), null); Statement stmt = conn.createStatement() @@ -615,14 +572,35 @@ public void runDml(IgniteBiTuple> qrysInfo, boolean i for (String cmd : cmds) stmt.execute(cmd); } - } - else if (type == DmlQueryType.SQL_FIELDS) { + catch (SQLException e) { + new RuntimeException(e); + } + }); + } + + /** + * @param qrysInfo DML commands info (queries, simple query flag). + */ + public void runSqlFieldsDml(IgniteBiTuple, Boolean> qrysInfo) { + executeDml(qrysInfo, (cmds) -> { IgniteCache cache = queryNode().getOrCreateCache("A"); cmds.forEach(c -> cache.query(new SqlFieldsQuery(c).setLocal(loc))); - } + }); + } + + /** + * @param qrysInfo DML commands info (queries, simple query flag). + * @param task Task to be executed. + */ + public void executeDml(IgniteBiTuple, Boolean> qrysInfo, Consumer> task) { + assumeFalse("There is no lazy mode for DML operations", !isFullyFetched); - checkSqlPlanHistoryDml(3, isSimpleQry); + List cmds = qrysInfo.get1(); + + task.accept(cmds); + + checkSqlPlanHistoryDml(3, qrysInfo.get2()); } /** @@ -638,17 +616,15 @@ public Map getSqlPlanHistory(IgniteEx node) { * @param size Number of SQL plan entries expected to be in the history. */ public void checkSqlPlanHistory(int size) { - Map sqlPlans = getSqlPlanHistory(queryNode()); - - assertNotNull(sqlPlans); + executeHistoryCheck(size, plans -> { + if (!isReducePhase && isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { + assertEquals(0, plans.size()); - if (!isReducePhase && isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { - assertEquals(0, sqlPlans.size()); - - sqlPlans = getSqlPlanHistory(mapNode()); - } + plans = getSqlPlanHistory(mapNode()); + } - checkMetrics(size, sqlPlans); + return plans; + }); } /** @@ -658,24 +634,36 @@ public void checkSqlPlanHistory(int size) { * @param isSimpleQry Simple query flag. */ public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { - Map sqlPlans = getSqlPlanHistory(queryNode()); + executeHistoryCheck(size, plans -> { + if (sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { + String check; - assertNotNull(sqlPlans); + if (isSimpleQry) + check = "no SELECT queries have been executed."; + else + check = "the following " + (loc ? "local " : "") + "query has been executed:"; - if (sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { - String check; + plans = plans.entrySet().stream() + .filter(e -> e.getKey().plan().contains(check)) + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - if (isSimpleQry) - check = "no SELECT queries have been executed."; - else - check = "the following " + (loc ? "local " : "") + "query has been executed:"; + checkMetrics((!isSimpleQry && !loc) ? size : 0, getSqlPlanHistory(mapNode())); + } - sqlPlans = sqlPlans.entrySet().stream() - .filter(e -> e.getKey().plan().contains(check)) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); + return plans; + }); + } - checkMetrics((!isSimpleQry && !loc) ? size : 0, getSqlPlanHistory(mapNode())); - } + /** + * @param size Number of SQL plan entries expected to be in the history. + * @param func Function to get the correct collection of plans for further checking. + */ + public void executeHistoryCheck(int size, Function, Map> func) { + Map sqlPlans = getSqlPlanHistory(queryNode()); + + assertNotNull(sqlPlans); + + sqlPlans = func.apply(sqlPlans); checkMetrics(size, sqlPlans); } @@ -757,13 +745,4 @@ public static int fail() { throw new IgniteSQLException("SQL function fail for test purpuses"); } } - - /** */ - private enum DmlQueryType { - /** JDBC query. */ - JDBC, - - /** SqlFields query. */ - SQL_FIELDS - } } From 04545927ec94567128552fd7384ed8e891108490 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Tue, 8 Oct 2024 09:14:51 +1000 Subject: [PATCH 31/33] IGNITE-22557 Changes from master merged --- .../ignite/testsuites/IgniteCalciteTestSuite.java | 11 ----------- .../ignite/testsuites/IntegrationTestSuite.java | 1 - .../internal/processors/query/running/SqlPlan.java | 1 - .../internal/processors/query/h2/H2QueryInfo.java | 5 ----- .../ignite/testsuites/IgniteSpringTestSuite.java | 2 +- 5 files changed, 1 insertion(+), 19 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java index 757f233f42854..e71990733db80 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java @@ -18,11 +18,6 @@ package org.apache.ignite.testsuites; import org.apache.ignite.internal.processors.query.calcite.QueryCheckerTest; -import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryCalciteFromClientSelfTest; -import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryCalciteSelfTest; -import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryConfigTest; -import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryH2FromClientSelfTest; -import org.apache.ignite.internal.processors.query.calcite.SqlPlanHistoryH2SelfTest; import org.apache.ignite.internal.processors.query.calcite.exec.ClosableIteratorsHolderTest; import org.apache.ignite.internal.processors.query.calcite.exec.LogicalRelImplementorTest; import org.apache.ignite.internal.processors.query.calcite.exec.NumericTypesPrecisionsTest; @@ -55,12 +50,6 @@ CalciteCommunicationMessageSerializationTest.class, NumericTypesPrecisionsTest.class, - - SqlPlanHistoryConfigTest.class, - SqlPlanHistoryCalciteSelfTest.class, - SqlPlanHistoryCalciteFromClientSelfTest.class, - SqlPlanHistoryH2SelfTest.class, - SqlPlanHistoryH2FromClientSelfTest.class, }) public class IgniteCalciteTestSuite { } diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index 1514c2f203c63..1aa5e6eb3ba8c 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -140,7 +140,6 @@ DdlTransactionCalciteSelfTest.class, MultiLineQueryTest.class, ViewsIntegrationTest.class, - SqlPlanHistoryConfigIntegrationTest.class, SqlPlanHistoryIntegrationTest.class, }) public class IntegrationTestSuite { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java index b77d67e919507..5e1876daafafa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/running/SqlPlan.java @@ -19,7 +19,6 @@ import java.util.Objects; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; /** Representation of an entry in SQL plan history. */ public class SqlPlan { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java index 64f7efc26b3ab..95ce6c1bf7e64 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java @@ -121,11 +121,6 @@ public String plan() { return stmt.getPlanSQL(); } - /** */ - public long beginTs() { - return beginTs; - } - /** */ public long extWait() { return extWait; diff --git a/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java b/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java index 50ac65e09125f..36e7b4c2937d9 100644 --- a/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java +++ b/modules/spring/src/test/java/org/apache/ignite/testsuites/IgniteSpringTestSuite.java @@ -88,7 +88,7 @@ // CDC tests. CdcConfigurationTest.class, - SqlPlanHistoryConfigTest.class + SqlPlanHistoryConfigTest.class, }) public class IgniteSpringTestSuite { } From 183af7f1537a4ef785ef08a3c6b95fe9952f8777 Mon Sep 17 00:00:00 2001 From: 21518201 Date: Tue, 8 Oct 2024 14:07:38 +1000 Subject: [PATCH 32/33] IGNITE-22557 Changes for failed tests --- .../calcite/integration/SqlPlanHistoryIntegrationTest.java | 2 +- .../org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java | 3 ++- .../java/org/apache/ignite/util/SystemViewCommandTest.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 2c77a91ea2c80..2bca301a48a37 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -471,7 +471,7 @@ public void testEntryReplacement() throws IgniteInterruptedCheckedException { timeStamps[i] = getSqlPlanHistory(queryNode()).values().stream().findFirst().get(); - U.sleep(10); + U.sleep(1000); } assertTrue(timeStamps[1] > timeStamps[0]); diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java index a8cf5bc812e72..e416d417bda9e 100755 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMetadataSelfTest.java @@ -367,7 +367,8 @@ public void testGetAllView() throws Exception { "STATISTICS_CONFIGURATION", "STATISTICS_PARTITION_DATA", "STATISTICS_LOCAL_DATA", - "STATISTICS_GLOBAL_DATA" + "STATISTICS_GLOBAL_DATA", + "SQL_PLANS_HISTORY" )); Set actViews = new TreeSet<>(); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java index 95a8e79a84fcc..d0259243379c6 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/SystemViewCommandTest.java @@ -495,7 +495,8 @@ public void testViews() { "DS_SETS", "DS_SEMAPHORES", "DS_QUEUES", - "PAGES_TIMESTAMP_HISTOGRAM" + "PAGES_TIMESTAMP_HISTOGRAM", + "SQL_PLANS_HISTORY" )); Set viewNames = new TreeSet<>(); From 6ab7de644a8300d4ec8330dc974c3f766d01f6a1 Mon Sep 17 00:00:00 2001 From: oleg-vlsk Date: Tue, 8 Oct 2024 15:15:16 +1000 Subject: [PATCH 33/33] IGNITE-22557 SqlPlanHistoryIntegrationTest refactored --- .../SqlPlanHistoryIntegrationTest.java | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java index 2c77a91ea2c80..06582c4135bc6 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlPlanHistoryIntegrationTest.java @@ -220,7 +220,7 @@ private CacheConfiguration configureCache(String name, Class... idxTypes) { * @return Ignite node where queries are executed. */ protected IgniteEx queryNode() { - IgniteEx node = isClient ? grid(2) : grid(0); + IgniteEx node = isClient ? grid(1) : grid(0); if (isClient) assertTrue(node.context().clientNode()); @@ -230,27 +230,16 @@ protected IgniteEx queryNode() { return node; } - /** - * @return Ignite map node. - */ - protected IgniteEx mapNode() { - IgniteEx node = grid(1); - - assertFalse(node.context().clientNode()); - - return node; - } - /** * Starts Ignite instance. * * @throws Exception In case of failure. */ protected void startTestGrid() throws Exception { - startGrids(2); + startGrid(0); if (isClient) - startClientGrid(2); + startClientGrid(1); } /** {@inheritDoc} */ @@ -620,7 +609,7 @@ public void checkSqlPlanHistory(int size) { if (!isReducePhase && isClient && sqlEngine == IndexingQueryEngineConfiguration.ENGINE_NAME) { assertEquals(0, plans.size()); - plans = getSqlPlanHistory(mapNode()); + plans = getSqlPlanHistory(grid(0)); } return plans; @@ -646,8 +635,6 @@ public void checkSqlPlanHistoryDml(int size, boolean isSimpleQry) { plans = plans.entrySet().stream() .filter(e -> e.getKey().plan().contains(check)) .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - - checkMetrics((!isSimpleQry && !loc) ? size : 0, getSqlPlanHistory(mapNode())); } return plans;