From fa195b26d451503889c5c41c7b5e4d8c55a48109 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 16 Jul 2024 10:13:20 +0200 Subject: [PATCH 01/10] LDEV-5023 test case for qoq regression https://luceeserver.atlassian.net/browse/LDEV-5023 --- .../java/lucee/runtime/db/HSQLDBHandler.java | 24 ++-- test/tickets/LDEV5023.cfc | 105 ++++++++++++++++++ 2 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 test/tickets/LDEV5023.cfc diff --git a/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java b/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java index afaf7cc9ea..2c29a5b927 100644 --- a/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java +++ b/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java @@ -61,7 +61,7 @@ import lucee.runtime.type.StructImpl; import lucee.runtime.type.dt.TimeSpan; import lucee.runtime.type.util.CollectionUtil; - +import lucee.aprint; /** * class to reexecute queries on the resultset object inside the cfml environment */ @@ -374,6 +374,8 @@ private static Struct getUsedColumnsForQuery(Connection conn, SQL sql) { // TODO consider if worth doing, if recordcount / column count is too small + aprint.o(sql.toString()); + try { Statement stat = conn.createStatement(); stat.execute("CREATE VIEW " + view + " AS " + sql.toString()); // + StringUtil.toUpperCase(sql.toString())); @@ -403,8 +405,8 @@ private static Struct getUsedColumnsForQuery(Connection conn, SQL sql) { Struct tableCols = ((Struct) tables.get(tableName)); tableCols.setEL(Caster.toKey(rs.getString(colPos)), null); } - // aprint.o(rs); - // aprint.o(tables); + aprint.o(rs); + aprint.o(tables); // don't need the view anymore, bye bye stat.execute("DROP VIEW " + view); } @@ -569,19 +571,27 @@ public static QueryImpl __execute(PageContext pc, SQL sql, int maxrows, int fetc try { // we now only lock the data loading, not the execution of the query, but should this be done via // cflock by the developer? + aprint.out(tables); synchronized (lock) { Iterator it = tables.iterator(); String cfQueryName = null; // name of the source query variable String dbTableName = null; // name of the table in the database String modSql = null; // int len=tables.size(); + SystemOut.print("QoQ HSQLDB CREATED TABLES: " + sql.toString()); while (it.hasNext()) { cfQueryName = it.next().toString();// tables.get(i).toString(); dbTableName = cfQueryName.replace('.', '_'); + if (qoqTables.contains(dbTableName)){ + aprint.o("duplicate table name!!"); + } - // this could match the wrong strings?? - modSql = StringUtil.replace(sql.getSQLString(), cfQueryName, dbTableName, false); - sql.setSQLString(modSql); + if (!cfQueryName.toLowerCase().equals(dbTableName.toLowerCase())){ + // TODO this could match the wrong strings?? + modSql = StringUtil.replace(sql.getSQLString(), cfQueryName, dbTableName, false); + sql.setSQLString(modSql); + SystemOut.print("QoQ HSQLDB CREATED TABLES: " + modSql); + } if (sql.getItems() != null && sql.getItems().length > 0) sql = new SQLImpl(sql.toString()); // temp tables still get created will all the source columns, // only populateTables is driven by the required columns calculated from the view @@ -589,7 +599,7 @@ public static QueryImpl __execute(PageContext pc, SQL sql, int maxrows, int fetc qoqTables.add(dbTableName); } - // SystemOut.print("QoQ HSQLDB CREATED TABLES: " + sql.toString()); + // create the sql as a view, to find out which table columns are needed Struct allTableColumns = getUsedColumnsForQuery(conn, sql); diff --git a/test/tickets/LDEV5023.cfc b/test/tickets/LDEV5023.cfc new file mode 100644 index 0000000000..043eaf74dd --- /dev/null +++ b/test/tickets/LDEV5023.cfc @@ -0,0 +1,105 @@ +component extends="org.lucee.cfml.test.LuceeTestCase" labels="qoq" { + + function run( testResults , testBox ) { + + describe( title='QofQ' , body=function(){ + + it( title='QoQ select * from table same source table name HSQLDB', body=function() { + var q = extensionList(); + var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word + // native engine + cols = "name, id"; + systemOutput("native [#cols#]", true); + var q_native = QueryExecute( + sql = "SELECT #cols# FROM q", + options = { dbtype: 'query', maxrows=5 } + ); + var q_stash = duplicate( q_native ); + // hsqldb engine, coz join + systemOutput("hsqldb", true); + var q_hsqlb = QueryExecute( + sql = "SELECT t1.name FROM q_native t1, q_native t2 WHERE t1.id = t2.id", + options = { dbtype: 'query' } + ); + systemOutput( q_hsqlb, true ); + expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_stash.recordcount ).toBe( q_native.recordcount ); + }); + + it( title='QoQ select * from table same source table name (arguments) HSQLDB', body=function() { + var q = extensionList(); + var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word + // native engine + cols = "name, id"; + systemOutput("native [#cols#]", true); + var q_native = QueryExecute( + sql = "SELECT #cols# FROM q", + options = { dbtype: 'query', maxrows=5 } + ); + var q_stash = duplicate( q_native ); + // hsqldb engine, coz join + systemOutput("hsqldb", true); + arguments.q_native = q_native; + var q_hsqlb = QueryExecute( + sql = "SELECT t1.name FROM q_native t1, arguments.q_native t2 WHERE t1.id = t2.id", + options = { dbtype: 'query' } + ); + systemOutput( q_hsqlb, true ); + expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_stash.recordcount ).toBe( q_native.recordcount ); + }); + + it( title='QoQ select * from table same source table name (all cols) HSQLDB', body=function() { + var q = extensionList(); + var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word + // native engine + systemOutput("native [#cols#]", true); + var q_native = QueryExecute( + sql = "SELECT #cols# FROM q", + options = { dbtype: 'query', maxrows=5 } + ); + var q_stash = duplicate( q_native ); + // hsqldb engine, coz join + systemOutput("hsqldb", true); + var q_hsqlb = QueryExecute( + sql = "SELECT t1.name FROM q_native t1, q_native t2 WHERE t1.id = t2.id", + options = { dbtype: 'query' } + ); + systemOutput( q_hsqlb, true ); + expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); + expect( q_stash.recordcount ).toBe( q_native.recordcount ); + }); + + it( title='QoQ select * from table same source table name (all cols) HSQLDB', body=function() { + var q = extensionList(); + var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word + // native engine + q = QueryExecute( + sql = "SELECT #cols# FROM q", + options = { dbtype: 'query' } + ); + // hsqldb engine, coz join + q = QueryExecute( + sql = "SELECT t1.name FROM q t1, q t2 WHERE t1.id = t2.id", + options = { dbtype: 'query' } + ); + }); + }); + + } + + private function getDummyData (){ + var q = queryNew("id,name,data","integer,varchar, varchar"); + loop list="micha,zac,brad,pothys,gert" item="n" index="i" { + var r = queryAddRow(q); + querySetCell(q, "id", r, r) + querySetCell(q, "name", n, r) + querySetCell(q, "data", repeatString("lucee",1000), r); + } + return q; + } + +} \ No newline at end of file From 874d789e5e33d70eaf6b3deb9f655e361e5cc2c8 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 16 Jul 2024 11:07:15 +0200 Subject: [PATCH 02/10] LDEV-5023 use a lock around entire QoQ jdbc process --- .../java/lucee/runtime/db/HSQLDBHandler.java | 92 +++++++------------ test/tickets/LDEV5023.cfc | 60 ++++++------ 2 files changed, 66 insertions(+), 86 deletions(-) diff --git a/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java b/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java index 2c29a5b927..b75ec6ad30 100644 --- a/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java +++ b/core/src/main/java/lucee/runtime/db/HSQLDBHandler.java @@ -61,7 +61,7 @@ import lucee.runtime.type.StructImpl; import lucee.runtime.type.dt.TimeSpan; import lucee.runtime.type.util.CollectionUtil; -import lucee.aprint; + /** * class to reexecute queries on the resultset object inside the cfml environment */ @@ -301,7 +301,6 @@ private static String toUsableType(int type) { * @throws DatabaseException */ private static void removeTable(Connection conn, String name) throws SQLException { - name = name.replace('.', '_'); Statement stat = conn.createStatement(); stat.execute("DROP TABLE " + name); DBUtil.commitEL(conn); @@ -362,11 +361,8 @@ private static void _executeStatement(Connection conn, String sql) throws SQLExc * @throws DatabaseException */ private static Struct getUsedColumnsForQuery(Connection conn, SQL sql) { - - // TODO this could be potentially cached against the sql text - - Stopwatch stopwatch = new Stopwatch(Stopwatch.UNIT_MILLI); - stopwatch.start(); + // Stopwatch stopwatch = new Stopwatch(Stopwatch.UNIT_MILLI); + // stopwatch.start(); ResultSet rs = null; ResultSetMetaData rsmd = null; String view = "V_QOQ_TEMP"; @@ -374,8 +370,6 @@ private static Struct getUsedColumnsForQuery(Connection conn, SQL sql) { // TODO consider if worth doing, if recordcount / column count is too small - aprint.o(sql.toString()); - try { Statement stat = conn.createStatement(); stat.execute("CREATE VIEW " + view + " AS " + sql.toString()); // + StringUtil.toUpperCase(sql.toString())); @@ -405,8 +399,8 @@ private static Struct getUsedColumnsForQuery(Connection conn, SQL sql) { Struct tableCols = ((Struct) tables.get(tableName)); tableCols.setEL(Caster.toKey(rs.getString(colPos)), null); } - aprint.o(rs); - aprint.o(tables); + // aprint.o(rs); + // aprint.o(tables); // don't need the view anymore, bye bye stat.execute("DROP VIEW " + view); } @@ -469,7 +463,6 @@ public QueryImpl execute(PageContext pc, final SQL sql, int maxrows, int fetchsi } catch (Exception ex) { } - } catch (Exception e) { qoqException = e; @@ -478,7 +471,6 @@ public QueryImpl execute(PageContext pc, final SQL sql, int maxrows, int fetchsi // If our first pass at the QoQ failed, lets look at the exception to see what we want to do with // it. if (qoqException != null) { - // Track the root cause Exception rootCause = qoqException; @@ -559,38 +551,29 @@ public static QueryImpl __execute(PageContext pc, SQL sql, int maxrows, int fetc ConfigPro config = (ConfigPro) pc.getConfig(); DatasourceConnection dc = null; Connection conn = null; - try { - DatasourceConnPool pool = config.getDatasourceConnectionPool(config.getDataSource(QOQ_DATASOURCE_NAME), "sa", ""); - dc = pool.borrowObject(); - conn = dc.getConnection(); - - // executeStatement(conn, "CONNECT"); // create a new HSQLDB session for temp tables - DBUtil.setAutoCommitEL(conn, false); - - // sql.setSQLString(HSQLUtil.sqlToZQL(sql.getSQLString(),false)); + // TODO this is currently single threaded + synchronized (lock) { try { - // we now only lock the data loading, not the execution of the query, but should this be done via - // cflock by the developer? - aprint.out(tables); - synchronized (lock) { + DatasourceConnPool pool = config.getDatasourceConnectionPool(config.getDataSource(QOQ_DATASOURCE_NAME), "sa", ""); + dc = pool.borrowObject(); + conn = dc.getConnection(); + // executeStatement(conn, "CONNECT"); // TODO create a new HSQLDB session for temp tables + DBUtil.setAutoCommitEL(conn, false); + + try { Iterator it = tables.iterator(); - String cfQueryName = null; // name of the source query variable - String dbTableName = null; // name of the table in the database + String cfQueryName = null; // name of the source cfml query variable + String dbTableName = null; // name of the target table in the database String modSql = null; // int len=tables.size(); - SystemOut.print("QoQ HSQLDB CREATED TABLES: " + sql.toString()); while (it.hasNext()) { cfQueryName = it.next().toString();// tables.get(i).toString(); dbTableName = cfQueryName.replace('.', '_'); - if (qoqTables.contains(dbTableName)){ - aprint.o("duplicate table name!!"); - } if (!cfQueryName.toLowerCase().equals(dbTableName.toLowerCase())){ // TODO this could match the wrong strings?? modSql = StringUtil.replace(sql.getSQLString(), cfQueryName, dbTableName, false); sql.setSQLString(modSql); - SystemOut.print("QoQ HSQLDB CREATED TABLES: " + modSql); } if (sql.getItems() != null && sql.getItems().length > 0) sql = new SQLImpl(sql.toString()); // temp tables still get created will all the source columns, @@ -599,7 +582,7 @@ public static QueryImpl __execute(PageContext pc, SQL sql, int maxrows, int fetc qoqTables.add(dbTableName); } - + // SystemOut.print("QoQ HSQLDB CREATED TABLES: " + sql.toString()); // create the sql as a view, to find out which table columns are needed Struct allTableColumns = getUsedColumnsForQuery(conn, sql); @@ -640,32 +623,27 @@ public static QueryImpl __execute(PageContext pc, SQL sql, int maxrows, int fetc } } + catch (SQLException e) { + throw (IllegalQoQException) (new IllegalQoQException("QoQ HSQLDB: error executing sql statement on query.", e.getMessage(), sql, null).initCause(e)); + } } - catch (SQLException e) { - throw (IllegalQoQException) (new IllegalQoQException("QoQ HSQLDB: error executing sql statement on query.", e.getMessage(), sql, null).initCause(e)); - // DatabaseException de = new DatabaseException("QoQ HSQLDB: error executing sql statement on query - // [" + e.getMessage() + "]", null , sql, null); - // throw de; - } - } - catch (Exception ee) { - throw (IllegalQoQException) (new IllegalQoQException("QoQ HSQLDB: error executing sql statement on query.", ee.getMessage(), sql, null).initCause(ee)); - // DatabaseException de = new DatabaseException("QoQ HSQLDB: error executing sql statement on query - // [" + ee.getMessage() + "]", null , sql, null); - // throw ee; - } - finally { - if (conn != null) { - removeAll(conn, qoqTables); - // executeStatement(conn, "DISCONNECT"); // close HSQLDB session with temp tables - DBUtil.setAutoCommitEL(conn, true); + catch (Exception ee) { + throw (IllegalQoQException) (new IllegalQoQException("QoQ HSQLDB: error executing sql statement on query.", ee.getMessage(), sql, null).initCause(ee)); } - if (dc != null) ((DatasourceConnectionPro) dc).release(); + finally { + if (conn != null) { + removeAll(conn, qoqTables); + //executeStatement(conn, "DISCONNECT"); // close HSQLDB session with temp tables + DBUtil.setAutoCommitEL(conn, true); + } + if (dc != null) ((DatasourceConnectionPro) dc).release(); - // manager.releaseConnection(dc); + // manager.releaseConnection(dc); + } + // TODO we are swallowing errors, shouldn't be passing a null value back + if (nqr != null) nqr.setExecutionTime(stopwatch.time()); + return nqr; } - // TOOD we are swalloing errors, shouldn't be passing a null value bacl - if (nqr != null) nqr.setExecutionTime(stopwatch.time()); - return nqr; + } } \ No newline at end of file diff --git a/test/tickets/LDEV5023.cfc b/test/tickets/LDEV5023.cfc index 043eaf74dd..599dabf2a5 100644 --- a/test/tickets/LDEV5023.cfc +++ b/test/tickets/LDEV5023.cfc @@ -5,101 +5,103 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="qoq" { describe( title='QofQ' , body=function(){ it( title='QoQ select * from table same source table name HSQLDB', body=function() { - var q = extensionList(); + var q = getTestQuery(); var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word // native engine cols = "name, id"; - systemOutput("native [#cols#]", true); var q_native = QueryExecute( sql = "SELECT #cols# FROM q", options = { dbtype: 'query', maxrows=5 } ); var q_stash = duplicate( q_native ); + expect( q_stash.recordcount ).toBe( q_native.recordcount ); // hsqldb engine, coz join - systemOutput("hsqldb", true); var q_hsqlb = QueryExecute( sql = "SELECT t1.name FROM q_native t1, q_native t2 WHERE t1.id = t2.id", options = { dbtype: 'query' } ); - systemOutput( q_hsqlb, true ); expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_stash.recordcount ).toBe( q_native.recordcount ); }); it( title='QoQ select * from table same source table name (arguments) HSQLDB', body=function() { - var q = extensionList(); + var q = getTestQuery(); var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word // native engine cols = "name, id"; - systemOutput("native [#cols#]", true); var q_native = QueryExecute( sql = "SELECT #cols# FROM q", options = { dbtype: 'query', maxrows=5 } ); var q_stash = duplicate( q_native ); // hsqldb engine, coz join - systemOutput("hsqldb", true); arguments.q_native = q_native; var q_hsqlb = QueryExecute( sql = "SELECT t1.name FROM q_native t1, arguments.q_native t2 WHERE t1.id = t2.id", options = { dbtype: 'query' } ); - systemOutput( q_hsqlb, true ); + expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_stash.recordcount ).toBe( q_native.recordcount ); }); it( title='QoQ select * from table same source table name (all cols) HSQLDB', body=function() { - var q = extensionList(); + var q = getTestQuery(); var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word // native engine - systemOutput("native [#cols#]", true); var q_native = QueryExecute( sql = "SELECT #cols# FROM q", options = { dbtype: 'query', maxrows=5 } ); var q_stash = duplicate( q_native ); // hsqldb engine, coz join - systemOutput("hsqldb", true); var q_hsqlb = QueryExecute( sql = "SELECT t1.name FROM q_native t1, q_native t2 WHERE t1.id = t2.id", options = { dbtype: 'query' } ); - systemOutput( q_hsqlb, true ); expect( q_stash.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_native.recordcount ).toBe( q_hsqlb.recordcount ); expect( q_stash.recordcount ).toBe( q_native.recordcount ); }); - it( title='QoQ select * from table same source table name (all cols) HSQLDB', body=function() { - var q = extensionList(); - var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word - // native engine - q = QueryExecute( - sql = "SELECT #cols# FROM q", - options = { dbtype: 'query' } - ); - // hsqldb engine, coz join - q = QueryExecute( - sql = "SELECT t1.name FROM q t1, q t2 WHERE t1.id = t2.id", - options = { dbtype: 'query' } - ); + it( title='QoQ select * from table same source table name (all cols) HSQLDB, 5000 threads', body=function() { + + var arr = []; + ArraySet(arr, 1, 1000, 0); + arrayEach(arr, function(){ + var q = getTestQuery(); + var cols = replaceNoCase( q.columnList, ",unique", "" ); // cleanup reserved word + // native engine + q = QueryExecute( + sql = "SELECT #cols# FROM q", + options = { dbtype: 'query' } + ); + // hsqldb engine, coz join + q = QueryExecute( + sql = "SELECT t1.name FROM q t1, q t2 WHERE t1.id = t2.id", + options = { dbtype: 'query' } + ); + }, true); + }); }); } - private function getDummyData (){ + private function getTestQuery(){ + return extensionList(); // has arrays and the like + /* var q = queryNew("id,name,data","integer,varchar, varchar"); - loop list="micha,zac,brad,pothys,gert" item="n" index="i" { - var r = queryAddRow(q); + loop list="micha,zac,brad,pothys,gert" item="local.n" index="local.i" { + var r = queryAddRow( q ); querySetCell(q, "id", r, r) querySetCell(q, "name", n, r) - querySetCell(q, "data", repeatString("lucee",1000), r); + //querySetCell(q, "data", repeatString("lucee",1000), r); } return q; + */ } } \ No newline at end of file From 5ff16c5ed5b541a146432c01dc0103bbf194f50f Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Thu, 18 Jul 2024 17:06:00 +0200 Subject: [PATCH 03/10] LDEV-5023 - set version --- loader/build.xml | 2 +- loader/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/build.xml b/loader/build.xml index 5add0194cc..b9350bf7ec 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index ff99ed79f6..9c38b24a1d 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.47-SNAPSHOT + 6.1.1.48-SNAPSHOT jar Lucee Loader Build From 85ad80a40db494c2f7269ce0e69a40c513812e5b Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Thu, 18 Jul 2024 17:17:19 +0200 Subject: [PATCH 04/10] 6.1.1.49 - improve exception message --- .../java/lucee/runtime/config/ConfigFactory.java | 12 ++++++++---- loader/build.xml | 2 +- loader/pom.xml | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/lucee/runtime/config/ConfigFactory.java b/core/src/main/java/lucee/runtime/config/ConfigFactory.java index a99af7644e..18dad3011d 100644 --- a/core/src/main/java/lucee/runtime/config/ConfigFactory.java +++ b/core/src/main/java/lucee/runtime/config/ConfigFactory.java @@ -220,14 +220,18 @@ static Struct loadDocumentCreateIfFails(Resource configFile, String type) throws catch (Exception e) { // rename buggy config files if (configFile.exists()) { - LogUtil.log(ThreadLocalPageContext.getConfig(), Log.LEVEL_INFO, ConfigFactory.class.getName(), - "Config file [" + configFile + "] was not valid and has been replaced"); - LogUtil.log(ThreadLocalPageContext.get(), ConfigFactory.class.getName(), e); - int count = 1; Resource bugFile; + int count = 1; Resource configDir = configFile.getParentResource(); while ((bugFile = configDir.getRealResource("lucee-" + type + "." + (count++) + ".buggy")).exists()) { } + + LogUtil.log(ThreadLocalPageContext.getConfig(), Log.LEVEL_INFO, ConfigFactory.class.getName(), + "The configuration file [" + configFile + + "] contained syntax errors and could not be read. A new configuration file has been created, and the invalid file has been renamed to [" + bugFile + + "]."); + LogUtil.log(ThreadLocalPageContext.get(), ConfigFactory.class.getName(), e); + IOUtil.copy(configFile, bugFile); configFile.delete(); } diff --git a/loader/build.xml b/loader/build.xml index b9350bf7ec..68c56551c2 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index 9c38b24a1d..4b16330049 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.48-SNAPSHOT + 6.1.1.49-SNAPSHOT jar Lucee Loader Build From a413f9bd6dc984201b317b302b0bf885945ec5a0 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Fri, 19 Jul 2024 12:23:31 +0200 Subject: [PATCH 05/10] add extension dir methods --- .../java/lucee/runtime/config/ConfigImpl.java | 43 +++++++++++++++---- .../java/lucee/runtime/config/ConfigPro.java | 5 +++ .../runtime/config/ConfigServerImpl.java | 5 +-- .../lucee/runtime/config/ConfigWebImpl.java | 10 +++++ .../config/SingleContextConfigWeb.java | 9 ++++ .../lucee/runtime/extension/RHExtension.java | 5 ++- loader/build.xml | 2 +- loader/pom.xml | 2 +- 8 files changed, 65 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/lucee/runtime/config/ConfigImpl.java b/core/src/main/java/lucee/runtime/config/ConfigImpl.java index c841443041..46fb776d32 100755 --- a/core/src/main/java/lucee/runtime/config/ConfigImpl.java +++ b/core/src/main/java/lucee/runtime/config/ConfigImpl.java @@ -2689,15 +2689,6 @@ public Resource getVideoDirectory() { return dir; } - @Override - public Resource getExtensionDirectory() { - // TODO take from tag - Resource dir = getConfigDir().getRealResource("extensions/installed"); - if (!dir.exists()) dir.mkdirs(); - - return dir; - } - @Override public ExtensionProvider[] getExtensionProviders() { throw new RuntimeException("no longer supported, use getRHExtensionProviders() instead."); @@ -3902,6 +3893,9 @@ boolean isEmpty(ClassDefinition cd) { private boolean fullNullSupport = false; + private Resource extInstalled; + private Resource extAvailable; + protected final void setFullNullSupport(boolean fullNullSupport) { this.fullNullSupport = fullNullSupport; } @@ -3990,4 +3984,35 @@ public int getReturnFormat() { protected void setReturnFormat(int returnFormat) { this.returnFormat = returnFormat; } + + @Override + public Resource getExtensionDirectory() { + return getExtensionInstalledDir(); + } + + @Override + public Resource getExtensionInstalledDir() { + if (extInstalled == null) { + synchronized (SystemUtil.createToken("extensions", "installed")) { + if (extInstalled == null) { + extInstalled = getConfigDir().getRealResource("extensions/installed"); + if (!extInstalled.exists()) extInstalled.mkdirs(); + } + } + } + return extInstalled; + } + + @Override + public Resource getExtensionAvailableDir() { + if (extAvailable == null) { + synchronized (SystemUtil.createToken("extensions", "available")) { + if (extAvailable == null) { + extAvailable = getConfigDir().getRealResource("extensions/available"); + if (!extAvailable.exists()) extAvailable.mkdirs(); + } + } + } + return extAvailable; + } } \ No newline at end of file diff --git a/core/src/main/java/lucee/runtime/config/ConfigPro.java b/core/src/main/java/lucee/runtime/config/ConfigPro.java index 31efbbd00e..1461954b4b 100644 --- a/core/src/main/java/lucee/runtime/config/ConfigPro.java +++ b/core/src/main/java/lucee/runtime/config/ConfigPro.java @@ -388,4 +388,9 @@ public Resource[] getResources(PageContext pc, Mapping[] mappings, String realPa public boolean getShowMetric(); public boolean getShowTest(); + + public Resource getExtensionInstalledDir(); + + public Resource getExtensionAvailableDir(); + } diff --git a/core/src/main/java/lucee/runtime/config/ConfigServerImpl.java b/core/src/main/java/lucee/runtime/config/ConfigServerImpl.java index d981ca5d77..21b3c19786 100644 --- a/core/src/main/java/lucee/runtime/config/ConfigServerImpl.java +++ b/core/src/main/java/lucee/runtime/config/ConfigServerImpl.java @@ -759,9 +759,7 @@ protected String getLibHash() { @Override public Resource getLocalExtensionProviderDirectory() { - Resource dir = getConfigDir().getRealResource("extensions/available"); - if (!dir.exists()) dir.mkdirs(); - return dir; + return getExtensionAvailableDir(); } protected void setAMFEngine(ClassDefinition cd, Map args) { @@ -873,4 +871,5 @@ public void setAdminMode(short adminMode) { public short getAdminMode() { return adminMode; } + } \ No newline at end of file diff --git a/core/src/main/java/lucee/runtime/config/ConfigWebImpl.java b/core/src/main/java/lucee/runtime/config/ConfigWebImpl.java index da6e320f27..30b2d97975 100644 --- a/core/src/main/java/lucee/runtime/config/ConfigWebImpl.java +++ b/core/src/main/java/lucee/runtime/config/ConfigWebImpl.java @@ -1921,4 +1921,14 @@ public boolean getFormUrlAsStruct() { public int getReturnFormat() { return instance.getReturnFormat(); } + + @Override + public Resource getExtensionInstalledDir() { + return instance.getExtensionInstalledDir(); + } + + @Override + public Resource getExtensionAvailableDir() { + return instance.getExtensionAvailableDir(); + } } diff --git a/core/src/main/java/lucee/runtime/config/SingleContextConfigWeb.java b/core/src/main/java/lucee/runtime/config/SingleContextConfigWeb.java index af7b123e03..4ab3e4f7c8 100644 --- a/core/src/main/java/lucee/runtime/config/SingleContextConfigWeb.java +++ b/core/src/main/java/lucee/runtime/config/SingleContextConfigWeb.java @@ -2101,4 +2101,13 @@ public int getReturnFormat() { return cs.getReturnFormat(); } + @Override + public Resource getExtensionInstalledDir() { + return cs.getExtensionInstalledDir(); + } + + @Override + public Resource getExtensionAvailableDir() { + return cs.getExtensionAvailableDir(); + } } \ No newline at end of file diff --git a/core/src/main/java/lucee/runtime/extension/RHExtension.java b/core/src/main/java/lucee/runtime/extension/RHExtension.java index 3bb4738bcb..60c11259eb 100644 --- a/core/src/main/java/lucee/runtime/extension/RHExtension.java +++ b/core/src/main/java/lucee/runtime/extension/RHExtension.java @@ -875,7 +875,8 @@ public static String toHash(String id, String version, String ext) { } public static Resource getExtensionInstalledDir(Config config) { - return config.getConfigDir().getRealResource("extensions/installed"); + + return ((ConfigPro) config).getExtensionInstalledDir(); } private static int getPhysicalExtensionCount(Config config) { @@ -894,7 +895,7 @@ public static void correctExtensions(Config config) throws PageException, IOExce // reduce the amount of extension stored in available { int max = 5; - Resource dir = config.getConfigDir().getRealResource("extensions/available"); + Resource dir = ((ConfigPro) config).getExtensionAvailableDir(); Resource[] resources = dir.listResources(LEX_FILTER); Map>> map = new HashMap<>(); RHExtension ext; diff --git a/loader/build.xml b/loader/build.xml index 68c56551c2..71f02244cd 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index 4b16330049..aa5d0062cf 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.49-SNAPSHOT + 6.1.1.50-SNAPSHOT jar Lucee Loader Build From aabc99c54cc8e91ab093c1875d30ee2efc7a15bd Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Fri, 19 Jul 2024 14:02:01 +0200 Subject: [PATCH 06/10] improve osgi file handling --- .../java/lucee/runtime/osgi/OSGiUtil.java | 75 +++++++++---------- loader/build.xml | 2 +- loader/pom.xml | 2 +- .../loader/engine/CFMLEngineFactory.java | 2 +- 4 files changed, 38 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java b/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java index c5927019ca..6325396376 100644 --- a/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java +++ b/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java @@ -837,7 +837,6 @@ private static String parentBundleText(String parentBundle) { private static Resource downloadBundle(CFMLEngineFactory factory, final String symbolicName, String symbolicVersion, Identification id) throws IOException, BundleException { resetJarsFromBundleDirectory(factory); - String strDownload = SystemUtil.getSystemPropOrEnvVar("lucee.enable.bundle.download", null); if (!Caster.toBooleanValue(strDownload, true)) { boolean printExceptions = Caster.toBooleanValue(SystemUtil.getSystemPropOrEnvVar("lucee.cli.printExceptions", null), false); @@ -859,7 +858,6 @@ private static Resource downloadBundle(CFMLEngineFactory factory, final String s final URL updateUrl = BundleProvider.getInstance().getBundleAsURL(new BundleDefinition(symbolicName, symbolicVersion), true); log(Logger.LOG_INFO, "Downloading bundle [" + symbolicName + ":" + symbolicVersion + "] from [" + updateUrl + "]"); - int code; HttpURLConnection conn; try { @@ -927,12 +925,12 @@ private static Resource downloadBundle(CFMLEngineFactory factory, final String s temp.delete(); } } - else { - Resource jar = jarDir.getRealResource(symbolicName + "-" + symbolicVersion + ".jar"); - IOUtil.copy((InputStream) conn.getContent(), jar, true); - conn.disconnect(); - return jar; - } + + Resource jar = jarDir.getRealResource(symbolicName + "-" + symbolicVersion + ".jar"); + IOUtil.copy((InputStream) conn.getContent(), jar, true); + conn.disconnect(); + return jar; + } /** @@ -1097,38 +1095,21 @@ private static BundleFile _getBundleFile(CFMLEngineFactory factory, BundleRange if (mbf != null) { return improveFileName(bd, mbf); } - List children = listFiles(dir, addional, JAR_EXT_FILTER); - - // now we check all jar files - { - - // now we check by Manifest comparsion, name not necessary reflect the correct bundle info - BundleFile bf; - for (boolean checkBundleRange: checkBundleRanges) { - mbf = null; - for (Resource child: children) { - if (checkBundleRange && !new Filter(bundleRange).accept(child.getName())) continue; - match = child; - bf = BundleFile.getInstance(child); - if (bf.isBundle()) { - if (bf.getSymbolicName().equals(bundleRange.getName())) { - if (bundleRange.matches(bf)) { - if (mbf == null || OSGiUtil.isNewerThan(bf.getVersion(), mbf.getVersion())) mbf = bf; - } - else { - if (versionsFound != null) { - if (versionsFound.length() > 0) versionsFound.append(", "); - versionsFound.append(bf.getVersionAsString()); - } - } - } - } - } - if (mbf != null) { - return improveFileName(factory.getBundleDirectory(), mbf); - } - } - } + /* + * List children = listFiles(dir, addional, JAR_EXT_FILTER); + * + * // now we check all jar files { + * + * // now we check by Manifest comparsion, name not necessary reflect the correct bundle info + * BundleFile bf; for (boolean checkBundleRange: checkBundleRanges) { mbf = null; for (Resource + * child: children) { if (checkBundleRange && !new Filter(bundleRange).accept(child.getName())) + * continue; match = child; bf = BundleFile.getInstance(child); if (bf.isBundle()) { if + * (bf.getSymbolicName().equals(bundleRange.getName())) { if (bundleRange.matches(bf)) { if (mbf == + * null || OSGiUtil.isNewerThan(bf.getVersion(), mbf.getVersion())) mbf = bf; } else { if + * (versionsFound != null) { if (versionsFound.length() > 0) versionsFound.append(", "); + * versionsFound.append(bf.getVersionAsString()); } } } } } if (mbf != null) { return + * improveFileName(factory.getBundleDirectory(), mbf); } } } + */ } catch (Exception e) { @@ -1154,6 +1135,20 @@ private static BundleFile _getBundleFile(CFMLEngineFactory factory, BundleRange return null; } + public static void correctBundles(CFMLEngineFactory factory) throws IOException, BundleException { + BundleFile bf; + String expName; + for (Resource child: ResourceUtil.toResource(factory.getBundleDirectory()).listResources(JAR_EXT_FILTER)) { + bf = BundleFile.getInstance(child); + if (bf.isBundle()) { + expName = bf.getSymbolicName() + "-" + bf.getVersionAsString() + ".jar"; + if (!child.getName().equals(expName)) { + child.moveTo(child.getParentResource().getRealResource(expName)); + } + } + } + } + /** * rename file to match the Manifest information * diff --git a/loader/build.xml b/loader/build.xml index 71f02244cd..a6df405ef4 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index aa5d0062cf..2cf0bf9fcb 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.50-SNAPSHOT + 6.1.1.51-SNAPSHOT jar Lucee Loader Build diff --git a/loader/src/main/java/lucee/loader/engine/CFMLEngineFactory.java b/loader/src/main/java/lucee/loader/engine/CFMLEngineFactory.java index 915c2535b9..c05020b554 100755 --- a/loader/src/main/java/lucee/loader/engine/CFMLEngineFactory.java +++ b/loader/src/main/java/lucee/loader/engine/CFMLEngineFactory.java @@ -982,7 +982,7 @@ public File downloadBundle(final String symbolicName, final String symbolicVersi + ", and has been prevented from downloading it. If this jar is not a core jar, it will need to be manually downloaded and placed in the {{lucee-server}}/context/bundles directory.")); } - jar = new File(jarDir, symbolicName.replace('.', '-') + "-" + symbolicVersion.replace('.', '-') + (".jar")); + jar = new File(jarDir, symbolicName + "-" + symbolicVersion + (".jar")); URL updateUrl; try { From 7eb3d22f30056040d25acc61deacd08b7f564766 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Fri, 19 Jul 2024 14:53:20 +0200 Subject: [PATCH 07/10] beta release --- loader/build.xml | 2 +- loader/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/build.xml b/loader/build.xml index a6df405ef4..c4da2e87f5 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index 2cf0bf9fcb..acf7547a4c 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.51-SNAPSHOT + 6.1.1.51-BETA jar Lucee Loader Build From 036dcc04ad67f776123a9dcad1a6a6b2278ead7d Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Sat, 20 Jul 2024 07:38:07 +0200 Subject: [PATCH 08/10] reduce rounds for createULID test case --- test/functions/CreateULID.cfc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functions/CreateULID.cfc b/test/functions/CreateULID.cfc index 044b7a860e..0e41e8e1a1 100644 --- a/test/functions/CreateULID.cfc +++ b/test/functions/CreateULID.cfc @@ -1,6 +1,6 @@ component extends="org.lucee.cfml.test.LuceeTestCase" { - variables.rounds = 10000; + variables.rounds = 10; variables.mysql = server.getDatasource("mysql"); function run( testResults , testBox ) { From 75305bfac80b8fcdd17d055b26cd2a232887b660 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Mon, 22 Jul 2024 11:49:28 +0200 Subject: [PATCH 09/10] partially invert "improve osgi file handling" --- .../java/lucee/runtime/osgi/OSGiUtil.java | 33 +++++++++++++++++++ loader/build.xml | 2 +- loader/pom.xml | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java b/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java index 6325396376..95da845433 100644 --- a/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java +++ b/core/src/main/java/lucee/runtime/osgi/OSGiUtil.java @@ -1095,6 +1095,39 @@ private static BundleFile _getBundleFile(CFMLEngineFactory factory, BundleRange if (mbf != null) { return improveFileName(bd, mbf); } + + List children = listFiles(dir, addional, JAR_EXT_FILTER); + + // now we check all jar files + { + + // now we check by Manifest comparsion, name not necessary reflect the correct bundle info + BundleFile bf; + for (boolean checkBundleRange: checkBundleRanges) { + mbf = null; + for (Resource child: children) { + if (checkBundleRange && !new Filter(bundleRange).accept(child.getName())) continue; + match = child; + bf = BundleFile.getInstance(child); + if (bf.isBundle()) { + if (bf.getSymbolicName().equals(bundleRange.getName())) { + if (bundleRange.matches(bf)) { + if (mbf == null || OSGiUtil.isNewerThan(bf.getVersion(), mbf.getVersion())) mbf = bf; + } + else { + if (versionsFound != null) { + if (versionsFound.length() > 0) versionsFound.append(", "); + versionsFound.append(bf.getVersionAsString()); + } + } + } + } + } + if (mbf != null) { + return improveFileName(factory.getBundleDirectory(), mbf); + } + } + } /* * List children = listFiles(dir, addional, JAR_EXT_FILTER); * diff --git a/loader/build.xml b/loader/build.xml index c4da2e87f5..a6df405ef4 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index acf7547a4c..2cf0bf9fcb 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.51-BETA + 6.1.1.51-SNAPSHOT jar Lucee Loader Build From 935eaff5d5f0cebf9a1082f9ca327e72e163a237 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Mon, 22 Jul 2024 12:30:57 +0200 Subject: [PATCH 10/10] beta release --- loader/build.xml | 2 +- loader/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/build.xml b/loader/build.xml index a6df405ef4..c4da2e87f5 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index 2cf0bf9fcb..acf7547a4c 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.51-SNAPSHOT + 6.1.1.51-BETA jar Lucee Loader Build