Merge pull request #65 from Open-EO/pagination

Support for pagination #64
Open-EO · Jan 4, 2025 · a5d56e3 · a5d56e3
2 parents f22f9a1 + 7665b60
commit a5d56e3
Show file tree

Hide file tree

Showing 7 changed files with 623 additions and 193 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - New function `getMissingBackends` for `Logs`
 - New property `federation:backends` added to the array returned by `validateProcess`
+- New functions in `Connection` for paginating through lists:
+  - `paginateProcesses`
+  - `paginateCollections`
+  - `paginateJobs`
+  - `paginateFiles`
+  - `paginateUserProcesses`
+  - `paginateServices`
+- The client may add a self link to the links in responses
+
+### Changed
+
+- The `listCollectionItems` function in `Connection` was completely rewritten.
 
 ## [2.6.0] - 2024-07-11
 

diff --git a/openeo.d.ts b/openeo.d.ts
@@ -2020,12 +2020,25 @@ declare module OpenEO {
          * List all collections available on the back-end.
          *
          * The collections returned always comply to the latest STAC version (currently 1.0.0).
+         * This function adds a self link to the response if not present.
          *
          * @async
          * @returns {Promise<Collections>} A response compatible to the API specification.
          * @throws {Error}
          */
         listCollections(): Promise<Collections>;
+        /**
+         * Paginate through the collections available on the back-end.
+         *
+         * The collections returned always complies to the latest STAC version (currently 1.0.0).
+         * This function adds a self link to the response if not present.
+         *
+         * @async
+         * @param {?number} [limit=50] - The number of collections per request/page as integer. If `null`, requests all collections.
+         * @yields {Promise<Collections>} A response compatible to the API specification.
+         * @throws {Error}
+         */
+        paginateCollections(limit?: number | null): AsyncGenerator<any, void, unknown>;
         /**
          * Get further information about a single collection.
          *
@@ -2039,12 +2052,11 @@ declare module OpenEO {
         describeCollection(collectionId: string): Promise<Collection>;
         /**
          * Loads items for a specific image collection.
+         *
          * May not be available for all collections.
          *
          * The items returned always comply to the latest STAC version (currently 1.0.0).
          *
-         * This is an experimental API and is subject to change.
-         *
          * @async
          * @param {string} collectionId - Collection ID to request items for.
          * @param {?Array.<number>} [spatialExtent=null] - Limits the items to the given bounding box in WGS84:
@@ -2076,20 +2088,40 @@ declare module OpenEO {
          */
         protected normalizeNamespace(namespace: string | null): string | null;
         /**
-         * List processes available on the back-end.
+         * List all processes available on the back-end.
          *
          * Requests pre-defined processes by default.
          * Set the namespace parameter to request processes from a specific namespace.
          *
          * Note: The list of namespaces can be retrieved by calling `listProcesses` without a namespace given.
          * The namespaces are then listed in the property `namespaces`.
          *
+         * This function adds a self link to the response if not present.
+         *
          * @async
          * @param {?string} [namespace=null] - Namespace of the processes (default to `null`, i.e. pre-defined processes). EXPERIMENTAL!
          * @returns {Promise<Processes>} - A response compatible to the API specification.
          * @throws {Error}
          */
         listProcesses(namespace?: string | null): Promise<Processes>;
+        /**
+         * Paginate through the processes available on the back-end.
+         *
+         * Requests pre-defined processes by default.
+         * Set the namespace parameter to request processes from a specific namespace.
+         *
+         * Note: The list of namespaces can be retrieved by calling `listProcesses` without a namespace given.
+         * The namespaces are then listed in the property `namespaces`.
+         *
+         * This function adds a self link to the response if not present.
+         *
+         * @async
+         * @param {?string} [namespace=null] - Namespace of the processes (default to `null`, i.e. pre-defined processes). EXPERIMENTAL!
+         * @param {?number} [limit=50] - The number of processes per request/page as integer. If `null`, requests all processes.
+         * @yields {Promise<Processes>} - A response compatible to the API specification.
+         * @throws {Error}
+         */
+        paginateProcesses(namespace?: string | null, limit?: number | null): AsyncGenerator<any, void, unknown>;
         /**
          * Get information about a single process.
          *
@@ -2239,13 +2271,22 @@ declare module OpenEO {
          */
         describeAccount(): Promise<UserAccount>;
         /**
-         * Lists all files from the user workspace.
+         * List all files from the user workspace.
          *
          * @async
          * @returns {Promise<ResponseArray.<UserFile>>} A list of files.
          * @throws {Error}
          */
         listFiles(): Promise<ResponseArray<UserFile>>;
+        /**
+         * Paginate through the files from the user workspace.
+         *
+         * @async
+         * @param {?number} [limit=50] - The number of files per request/page as integer. If `null`, requests all files.
+         * @yields {Promise<ResponseArray.<UserFile>>} A list of files.
+         * @throws {Error}
+         */
+        paginateFiles(limit?: number | null): AsyncGenerator<any, void, unknown>;
         /**
          * A callback that is executed on upload progress updates.
          *
@@ -2299,14 +2340,24 @@ declare module OpenEO {
          */
         validateProcess(process: Process): Promise<ValidationResult>;
         /**
-         * Lists all user-defined processes of the authenticated user.
+         * List all user-defined processes of the authenticated user.
          *
          * @async
          * @param {Array.<UserProcess>} [oldProcesses=[]] - A list of existing user-defined processes to update.
          * @returns {Promise<ResponseArray.<UserProcess>>} A list of user-defined processes.
          * @throws {Error}
          */
         listUserProcesses(oldProcesses?: Array<UserProcess>): Promise<ResponseArray<UserProcess>>;
+        /**
+         * Paginates through the user-defined processes of the authenticated user.
+         *
+         * @async
+         * @param {?number} [limit=50] - The number of processes per request/page as integer. If `null`, requests all processes.
+         * @param {Array.<UserProcess>} [oldProcesses=[]] - A list of existing user-defined processes to update.
+         * @yields {Promise<ResponseArray.<UserProcess>>} A list of user-defined processes.
+         * @throws {Error}
+         */
+        paginateUserProcesses(limit?: number | null, oldProcesses?: Array<UserProcess>): AsyncGenerator<any, void, unknown>;
         /**
          * Creates a new stored user-defined process at the back-end.
          *
@@ -2359,14 +2410,24 @@ declare module OpenEO {
          */
         downloadResult(process: Process, targetPath: string, plan?: string | null, budget?: number | null, abortController?: AbortController | null): Promise<void>;
         /**
-         * Lists all batch jobs of the authenticated user.
+         * List all batch jobs of the authenticated user.
          *
          * @async
          * @param {Array.<Job>} [oldJobs=[]] - A list of existing jobs to update.
          * @returns {Promise<ResponseArray.<Job>>} A list of jobs.
          * @throws {Error}
          */
         listJobs(oldJobs?: Array<Job>): Promise<ResponseArray<Job>>;
+        /**
+         * Paginate through the batch jobs of the authenticated user.
+         *
+         * @async
+         * @param {?number} [limit=50] - The number of jobs per request/page as integer. If `null`, requests all jobs.
+         * @param {Array.<Job>} [oldJobs=[]] - A list of existing jobs to update.
+         * @yields {Promise<ResponseArray.<Job>>} A list of jobs.
+         * @throws {Error}
+         */
+        paginateJobs(limit?: number | null, oldJobs?: Array<Job>): AsyncGenerator<any, void, unknown>;
         /**
          * Creates a new batch job at the back-end.
          *
@@ -2391,14 +2452,24 @@ declare module OpenEO {
          */
         getJob(id: string): Promise<Job>;
         /**
-         * Lists all secondary web services of the authenticated user.
+         * List all secondary web services of the authenticated user.
          *
          * @async
          * @param {Array.<Service>} [oldServices=[]] - A list of existing services to update.
          * @returns {Promise<ResponseArray.<Job>>} A list of services.
          * @throws {Error}
          */
         listServices(oldServices?: Array<Service>): Promise<ResponseArray<Job>>;
+        /**
+         * Paginate through the secondary web services of the authenticated user.
+         *
+         * @async
+         * @param {?number} [limit=50] - The number of services per request/page as integer. If `null` (default), requests all services.
+         * @param {Array.<Service>} [oldServices=[]] - A list of existing services to update.
+         * @yields {Promise<ResponseArray.<Job>>} A list of services.
+         * @throws {Error}
+         */
+        paginateServices(limit?: number | null, oldServices?: Array<Service>): AsyncGenerator<any, void, unknown>;
         /**
          * Creates a new secondary web service at the back-end.
          *
@@ -2433,9 +2504,10 @@ declare module OpenEO {
          * @protected
          * @param {Array.<*>} arr
          * @param {object.<string, *>} response
+         * @param {string} selfUrl
          * @returns {ResponseArray}
          */
-        protected _toResponseArray(arr: Array<any>, response: object<string, any>): ResponseArray;
+        protected _toResponseArray(arr: Array<any>, response: object<string, any>, selfUrl: string): ResponseArray;
         /**
          * Get the a link with the given rel type.
          *
@@ -2446,6 +2518,22 @@ declare module OpenEO {
          * @throws {Error}
          */
         protected _getLinkHref(links: Array<Link>, rel: string | Array<string>): string | null;
+        /**
+         * Get the URL of the next page from a response.
+         *
+         * @protected
+         * @param {AxiosResponse} response
+         * @returns {string | null}
+         */
+        protected _getNextLink(response: AxiosResponse): string | null;
+        /**
+         * Add a self link to the response if not present.
+         *
+         * @param {object} data - The body of the response as an object.
+         * @param {string} selfUrl - The URL of the current request.
+         * @returns {object} The modified object.
+         */
+        _addSelfLink(data: object, selfUrl: string): object;
         /**
          * Makes all links in the list absolute.
          *
@@ -2909,7 +2997,7 @@ declare module OpenEO {
     /**
      * An array, but enriched with additional details from an openEO API response.
      *
-     * Adds two properties: `links` and `federation:missing`.
+     * Adds three properties: `url`, `links` and `federation:missing`.
      */
     export type ResponseArray = any;
     export type ServiceType = object<string, any>;