add support for listeners to function S3ListBucket

build.number

@@ -1,3 +1,3 @@
 #Build Number for ANT. Do not edit!
-#Wed Nov 22 15:12:11 CET 2023
-build.number=13
+#Wed Nov 22 15:37:52 CET 2023
+build.number=14

source/fld/function.fld

@@ -989,6 +989,35 @@
 			<required>Yes</required>
 			<description>Name of the bucket to list objects from.</description>
 	    </argument>
+
+		<argument>
+			<name>listener</name>
+			<alias>closure,udf,function,component</alias>
+			<type>any</type>
+			<required>No</required>
+			<value>true</value>
+			<description>Optional. Defines the target where the downloaded data will be directed. 
+				If a file path is provided, the data is saved to that path, the file path must be provided with help of the function "fileOpen" like this [fileOpen(path,"write")]. 
+				If a closure or function is given, it will be invoked with parts of the downloaded data as its argument. 
+				The function should accept a single argument named 'line' for line-by-line processing, 
+				'string{Number}' for string blocks of a specified size, 
+				or 'binary{Number}' for binary blocks of a specified size. 
+
+				The function should return a boolean value: returning false will stop further reading from S3, 
+				while true will continue the process. 
+
+				If this argument is omitted, the function returns the downloaded data directly.</description>
+
+	    </argument>
+
+		<argument>
+			<name>blockfactor</name>
+			<alias>maxKeys,blockSize</alias>
+			<type>numeric</type>
+			<required>No</required>
+			<default>1000</default>
+			<description></description>
+	    </argument>
 		<argument>
 			<name>accessKeyId</name>
 			<alias>accessKey,awsAccessKeyId,awsAccessKey</alias>

source/java/src/org/lucee/extension/resource/s3/function/S3Download.java

@@ -69,13 +69,12 @@ public Object invoke(PageContext pc, Object[] args) throws PageException {
 			if (target instanceof UDF) {
 				targetUDF = (UDF) target;
 				validateInvoke(pc, targetUDF, mode, blockSize, false);
-
 			}
 			else if (target instanceof Component) {
 				targetCFC = (Component) target;
 				Component csa = toComponentSpecificAccess(Component.ACCESS_PRIVATE, targetCFC);
-				boolean hasBefore = toFunction(csa.get(BEFORE), null) != null;
-				boolean hasAfter = toFunction(csa.get(AFTER), null) != null;
+				boolean hasBefore = toFunction(csa.get(BEFORE, null), null) != null;
+				boolean hasAfter = toFunction(csa.get(AFTER, null), null) != null;
 				UDF invoke = toFunction(csa.get(INVOKE), null);
 				if (invoke == null) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 2, "component",
 						"the listener component does not contain a instance function with name [invoke] that is required", null);
@@ -218,7 +217,7 @@ else if (targetRes != null) {
 		}
 	}
 
-	private UDF toFunction(Object obj, UDF defaultValue) {
+	public static UDF toFunction(Object obj, UDF defaultValue) {
 		if (obj instanceof UDF) return (UDF) obj;
 		return defaultValue;
 	}

source/java/src/org/lucee/extension/resource/s3/function/S3ListBucket.java

@@ -1,45 +1,69 @@
 package org.lucee.extension.resource.s3.function;
 
 import org.lucee.extension.resource.s3.S3;
+import org.lucee.extension.resource.s3.listener.ComponentListListener;
+import org.lucee.extension.resource.s3.listener.ListListener;
+import org.lucee.extension.resource.s3.listener.UDFListListener;
 
 import lucee.loader.engine.CFMLEngine;
 import lucee.loader.engine.CFMLEngineFactory;
+import lucee.runtime.Component;
 import lucee.runtime.PageContext;
 import lucee.runtime.exp.PageException;
-import lucee.runtime.type.Query;
+import lucee.runtime.type.UDF;
 import lucee.runtime.util.Cast;
 
 public class S3ListBucket extends S3Function {
 
 	private static final long serialVersionUID = 3486553628255584848L;
 
-	public static Query call(PageContext pc, String bucketName, String accessKeyId, String secretAccessKey, String host, double timeout) throws PageException {
-
+	@Override
+	public Object invoke(PageContext pc, Object[] args) throws PageException {
 		CFMLEngine eng = CFMLEngineFactory.getInstance();
-		// for backward compatibility, when host was not existing
-		if (eng.getDecisionUtil().isNumber(host)) {
-			timeout = eng.getCastUtil().toDoubleValue(host);
-			host = null;
-		}
+		Cast cast = eng.getCastUtil();
+		if (args.length < 1 || args.length > 7) throw eng.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 1, 7, args.length);
+
+		// required
+		String bucketName = cast.toString(args[0]);
+
+		// optional
+		Object listener = args.length > 1 && args[1] != null ? args[1] : null;
+		int blockSize = args.length > 2 && args[2] != null ? cast.toIntValue(args[2]) : 1000;
+		String accessKeyId = args.length > 3 && args[3] != null ? cast.toString(args[3]) : null;
+		String secretAccessKey = args.length > 4 && args[4] != null ? cast.toString(args[4]) : null;
+		String host = args.length > 5 && args[5] != null ? cast.toString(args[5]) : null;
+		double timeout = args.length > 6 && !isEmpty(args[6]) ? cast.toDoubleValue(args[6]) : 0;
+
+		// validate
+
 		try {
 			S3 s3 = S3.getInstance(toS3Properties(pc, accessKeyId, secretAccessKey, host), toTimeout(timeout), pc.getConfig());
-			return s3.listObjectsAsQuery(bucketName);
 
+			// no listener
+			if (listener == null) {
+				return s3.listObjectsAsQuery(bucketName, blockSize, null);
+			}
+			else {
+				ListListener list = toListener(eng, pc, listener);
+				list.before();
+				s3.listObjectsAsQuery(bucketName, blockSize, toListener(eng, pc, listener));
+				list.after();
+			}
 		}
 		catch (Exception e) {
 			throw CFMLEngineFactory.getInstance().getCastUtil().toPageException(e);
 		}
+		return null;
 	}
 
-	@Override
-	public Object invoke(PageContext pc, Object[] args) throws PageException {
-		CFMLEngine engine = CFMLEngineFactory.getInstance();
-		Cast cast = engine.getCastUtil();
-		if (args.length == 5) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), cast.toString(args[3]), cast.toDoubleValue(args[4]));
-		if (args.length == 4) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), cast.toString(args[3]), 0);
-		if (args.length == 3) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), null, 0);
-		if (args.length == 2) return call(pc, cast.toString(args[0]), cast.toString(args[1]), null, null, 0);
-		if (args.length == 1) return call(pc, cast.toString(args[0]), null, null, null, 0);
-		throw engine.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 1, 5, args.length);
+	private ListListener toListener(CFMLEngine eng, PageContext pc, Object listener) throws PageException {
+		if (listener instanceof UDF) {
+			return new UDFListListener(eng, pc, (UDF) listener);
+		}
+		if (listener instanceof Component) {
+			return new ComponentListListener(eng, pc, (Component) listener);
+		}
+		throw CFMLEngineFactory.getInstance().getExceptionUtil().createFunctionException(pc, "S3ListBucket", 2, "listener",
+				"invalid listener type [" + listener.getClass().getName() + "], only functions and components are supported as listeners", "");
 	}
 }

source/java/src/org/lucee/extension/resource/s3/listener/ComponentListListener.java

@@ -0,0 +1,61 @@
+package org.lucee.extension.resource.s3.listener;
+
+import org.lucee.extension.resource.s3.function.S3Download;
+
+import lucee.loader.engine.CFMLEngine;
+import lucee.loader.util.Util;
+import lucee.runtime.Component;
+import lucee.runtime.PageContext;
+import lucee.runtime.exp.PageException;
+import lucee.runtime.type.Collection.Key;
+import lucee.runtime.type.Query;
+
+public class ComponentListListener implements ListListener {
+
+	private PageContext pc;
+	private Component listener;
+	private Component csa;
+	private Key INVOKE;
+	private Key BEFORE;
+	private Key AFTER;
+	private CFMLEngine eng;
+
+	public ComponentListListener(CFMLEngine eng, PageContext pc, Component listener) throws PageException {
+		INVOKE = eng.getCastUtil().toKey("invoke");
+		BEFORE = eng.getCastUtil().toKey("before");
+		AFTER = eng.getCastUtil().toKey("after");
+		this.eng = eng;
+		this.pc = pc;
+		this.listener = listener;
+		csa = S3Download.toComponentSpecificAccess(Component.ACCESS_PRIVATE, listener);
+
+	}
+
+	@Override
+	public void before() throws PageException {
+		if (S3Download.toFunction(csa.get(BEFORE, null), null) != null) {
+			listener.call(pc, BEFORE, new Object[] {});
+		}
+	}
+
+	@Override
+	public boolean invoke(Query data) throws PageException {
+		if (S3Download.toFunction(csa.get(INVOKE, null), null) != null) {
+			Object res = listener.call(pc, INVOKE, new Object[] { data });
+			if (res == null || Util.isEmpty(res.toString())) return true;
+			return eng.getCastUtil().toBooleanValue(res);
+		}
+		else {
+			throw eng.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 2, "component",
+					"the listener component does not contain a instance function with name [invoke] that is required", null);
+		}
+	}
+
+	@Override
+	public void after() throws PageException {
+		if (S3Download.toFunction(csa.get(AFTER, null), null) != null) {
+			listener.call(pc, AFTER, new Object[] {});
+		}
+	}
+
+}

source/java/src/org/lucee/extension/resource/s3/listener/ListListener.java

@@ -0,0 +1,14 @@
+package org.lucee.extension.resource.s3.listener;
+
+import lucee.runtime.exp.PageException;
+import lucee.runtime.type.Query;
+
+public interface ListListener {
+
+	public void before() throws PageException;
+
+	public boolean invoke(Query data) throws PageException;
+
+	public void after() throws PageException;
+
+}

source/java/src/org/lucee/extension/resource/s3/listener/UDFListListener.java

@@ -0,0 +1,37 @@
+package org.lucee.extension.resource.s3.listener;
+
+import lucee.loader.engine.CFMLEngine;
+import lucee.loader.util.Util;
+import lucee.runtime.PageContext;
+import lucee.runtime.exp.PageException;
+import lucee.runtime.type.Query;
+import lucee.runtime.type.UDF;
+
+public class UDFListListener implements ListListener {
+
+	private PageContext pc;
+	private UDF listener;
+	private CFMLEngine eng;
+
+	public UDFListListener(CFMLEngine eng, PageContext pc, UDF listener) {
+		this.eng = eng;
+		this.pc = pc;
+		this.listener = listener;
+	}
+
+	@Override
+	public boolean invoke(Query data) throws PageException {
+		Object res = listener.call(pc, new Object[] { data }, true);
+		if (res == null || Util.isEmpty(res.toString())) return true;
+		return eng.getCastUtil().toBooleanValue(res);
+	}
+
+	@Override
+	public void before() {
+	}
+
+	@Override
+	public void after() {
+	}
+
+}

tests/functions/S3ListBucket.cfc

@@ -3,18 +3,22 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" {
 		describe( title="Test suite for S3ListBucket()", body=function() {
 			it(title="check region with blackbaze",skip=Util::isBackBlazeNotSupported(), body = function( currentSpec ) {
 				testit(Util::getBackBlazeCredentials());
+				testUDF(Util::getBackBlazeCredentials());
 			});	
 
 			it(title="check with amazon",skip=Util::isAWSNotSupported(), body = function( currentSpec ) {
 				testit(Util::getAWSCredentials());
+				testUDF(Util::getAWSCredentials());
 			});	
 
 			it(title="check with wasabi",skip=Util::isWasabiNotSupported(), body = function( currentSpec ) {
 				testit(Util::getWasabiCredentials());
+				testUDF(Util::getWasabiCredentials());
 			});		
 
 			it(title="check with google",skip=Util::isGoogleNotSupported(), body = function( currentSpec ) {
 				testit(Util::getGoogleCredentials());
+				testUDF(Util::getGoogleCredentials());
 			});			
 
 		});
@@ -51,6 +55,56 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" {
 		}
 	}
 
+	private function testUDF(cred) {
+		try {
+			// create variables
+			var bucketName=cred.PREFIX&"-list-bucket:"&listFirst(replace(server.lucee.version,".","","all"),"-");
+			var objectNames=["sub/test1.txt","sub/test2.txt","sub/test3.txt","sub/test4.txt","sub/test5.txt"];
+
+			// create empty bucket
+			Util::deleteIfExists(cred,bucketName);
+			S3CreateBucket( 
+					bucketName:bucketName,
+					accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST));
+
+			// create objects
+			loop array=objectNames item="local.objectName" {
+				// create source bucket
+				if(!S3Exists( 
+					bucketName:bucketName,  objectName:objectName, 
+					accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST))) {
+					S3Write( 
+						value:"Susi Sorglos",
+						bucketName:bucketName,  objectName:objectName, 
+						accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST));
+				}
+			}
+
+			var res="";
+			S3ListBucket(
+				bucketName:bucketName, 
+				listener:function (data){
+					res&=data.recordcount&";";
+        			dump(arguments);
+				},
+				blockfactor:3,
+				accessKeyId:cred.ACCESS_KEY_ID, 
+				secretAccessKey:cred.SECRET_KEY,
+				host:(isNull(cred.HOST)?nullvalue():cred.HOST)
+			);
+			assertEquals("3;2;", res);
+		}
+		catch(e) {
+			if(!findNoCase("Transaction cap exceeded", e.message) ) throw e;
+		}
+		finally {
+			Util::deleteBucketEL(cred,bucketName);
+		}
+	}
+
+
+
+
 
 	private function doFind(value){
 		return value EQ "world";