merge: pull in changes from dev (#4300)

feat: merge dev into next 2024-09-30

apps/admin/src/App.auth0.tsx

@@ -8,11 +8,22 @@ export const App = () => {
     return (
         <Admin>
             <Auth0
+                autoLogin={() => {
+                    const query = new URLSearchParams(window.location.search);
+                    return query.get("action") !== "logout";
+                }}
                 auth0={{
                     domain: String(process.env.REACT_APP_AUTH0_DOMAIN),
                     clientId: String(process.env.REACT_APP_AUTH0_CLIENT_ID)
                 }}
                 rootAppClientId={String(process.env.REACT_APP_AUTH0_CLIENT_ID)}
+                onLogout={logout => {
+                    logout({
+                        logoutParams: {
+                            returnTo: window.location.origin + "?action=logout"
+                        }
+                    });
+                }}
             />
             <Extensions />
         </Admin>

apps/api/graphql/src/security.auth0.ts

@@ -78,7 +78,7 @@ export default ({ documentClient }: { documentClient: DynamoDBDocument }) => [
                 id: token["sub"],
                 type: "admin",
                 displayName: token["name"],
-                group: token["webiny_group"]
+                group: "full-access"
             };
         }
     }),

packages/api-aco/src/utils/decorators/decorateIfModelAuthorizationEnabled.ts

@@ -17,7 +17,7 @@ type FilterModelMethods<T> = {
  * E.g., `getEntryManager` has `model` typed as `CmsModel | string`.
  * Ideally, we would filter those out in the previous utility type, but I'm not sure how to achieve that.
  */
-type ModelMethods<T> = Omit<FilterModelMethods<T>, "getEntryManager">;
+type ModelMethods<T> = Omit<FilterModelMethods<T>, "getEntryManager" | "getSingletonEntryManager">;
 
 /**
  * Decorator takes the decoratee as the _first_ parameter, and then forwards the rest of the parameters.

packages/api-elasticsearch/src/plugins/operator/equal.ts

@@ -14,7 +14,7 @@ export class ElasticsearchQueryBuilderOperatorEqualPlugin extends ElasticsearchQ
     ): void {
         const { value, path, basePath } = params;
 
-        if (value === null) {
+        if (value === null || value === undefined) {
             query.must_not.push({
                 exists: {
                     field: path

packages/api-elasticsearch/src/plugins/operator/not.ts

@@ -14,7 +14,7 @@ export class ElasticsearchQueryBuilderOperatorNotPlugin extends ElasticsearchQue
     ): void {
         const { value, path, basePath } = params;
 
-        if (value === null) {
+        if (value === null || value === undefined) {
             query.filter.push({
                 exists: {
                     field: path

packages/api-elasticsearch/src/sort.ts

@@ -1,5 +1,5 @@
 import WebinyError from "@webiny/error";
-import { FieldSortOptions, SortType, SortOrder } from "~/types";
+import { FieldSortOptions, SortOrder, SortType } from "~/types";
 import { ElasticsearchFieldPlugin } from "~/plugins";
 
 const sortRegExp = new RegExp(/^([a-zA-Z-0-9_@]+)_(ASC|DESC)$/);
@@ -31,7 +31,7 @@ export const createSort = (params: CreateSortParams): SortType => {
     /**
      * Cast as string because nothing else should be allowed yet.
      */
-    return sort.reduce((acc, value) => {
+    const result = sort.reduce((acc, value) => {
         if (typeof value !== "string") {
             throw new WebinyError(`Sort as object is not supported..`);
         }
@@ -61,4 +61,13 @@ export const createSort = (params: CreateSortParams): SortType => {
 
         return acc;
     }, {} as Record<string, FieldSortOptions>);
+    /**
+     * If we do not have id in the sort, we add it as we need a tie_breaker for the Elasticsearch to be able to sort consistently.
+     */
+    if (!result["id.keyword"] && !result["id"]) {
+        result["id.keyword"] = {
+            order: "asc"
+        };
+    }
+    return result;
 };

packages/api-file-manager-s3/src/assetDelivery/assetDeliveryConfig.ts

@@ -15,6 +15,7 @@ export type AssetDeliveryParams = Parameters<typeof createBaseAssetDelivery>[0]
      * @see https://repost.aws/knowledge-center/presigned-url-s3-bucket-expiration
      */
     presignedUrlTtl?: number;
+    assetStreamingMaxSize?: number;
 };
 
 export const assetDeliveryConfig = (params: AssetDeliveryParams) => {
@@ -25,6 +26,11 @@ export const assetDeliveryConfig = (params: AssetDeliveryParams) => {
         // Presigned URLs last 1 hour
         presignedUrlTtl = 3600,
         imageResizeWidths = [100, 300, 500, 750, 1000, 1500, 2500],
+        /**
+         * Even though Lambda's response payload limit is 6,291,556 bytes, we leave some room for the response envelope.
+         * We had situations where a 4.7MB file would cause the payload to go over the limit, so let's be on the safe side.
+         */
+        assetStreamingMaxSize = 4718592,
         ...baseParams
     } = params;
 
@@ -41,7 +47,7 @@ export const assetDeliveryConfig = (params: AssetDeliveryParams) => {
             });
 
             config.decorateAssetOutputStrategy(() => {
-                return new S3OutputStrategy(s3, bucket, presignedUrlTtl);
+                return new S3OutputStrategy(s3, bucket, presignedUrlTtl, assetStreamingMaxSize);
             });
 
             config.decorateAssetTransformationStrategy(() => {

packages/api-file-manager-s3/src/assetDelivery/s3/S3OutputStrategy.ts

@@ -3,8 +3,6 @@ import { GetObjectCommand, getSignedUrl, S3 } from "@webiny/aws-sdk/client-s3";
 import { S3RedirectAssetReply } from "~/assetDelivery/s3/S3RedirectAssetReply";
 import { S3StreamAssetReply } from "~/assetDelivery/s3/S3StreamAssetReply";
 
-const MAX_RETURN_CONTENT_LENGTH = 4915200; // ~4.8 MB
-
 /**
  * This strategy outputs an asset taking into account the size of the asset contents.
  * If the asset is larger than 5MB, a presigned URL will be generated, and a redirect will happen.
@@ -13,21 +11,30 @@ export class S3OutputStrategy implements AssetOutputStrategy {
     private readonly s3: S3;
     private readonly bucket: string;
     private readonly presignedUrlTtl: number;
+    private readonly assetStreamingMaxSize: number;
 
-    constructor(s3: S3, bucket: string, presignedUrlTtl: number) {
+    constructor(s3: S3, bucket: string, presignedUrlTtl: number, assetStreamingMaxSize: number) {
+        this.assetStreamingMaxSize = assetStreamingMaxSize;
         this.presignedUrlTtl = presignedUrlTtl;
         this.s3 = s3;
         this.bucket = bucket;
     }
 
     async output(asset: Asset): Promise<AssetReply> {
-        if ((await asset.getSize()) > MAX_RETURN_CONTENT_LENGTH) {
+        if (asset.getSize() > this.assetStreamingMaxSize) {
+            console.log(
+                `Asset size is greater than ${this.assetStreamingMaxSize}; redirecting to a presigned S3 URL.`
+            );
+
             return new S3RedirectAssetReply(
                 await this.getPresignedUrl(asset),
                 this.presignedUrlTtl
             );
         }
 
+        console.log(
+            `Asset size is smaller than ${this.assetStreamingMaxSize}; streaming directly from Lambda function.`
+        );
         return new S3StreamAssetReply(asset);
     }
 

packages/api-file-manager-s3/src/assetDelivery/s3/SharpTransform.ts

@@ -26,6 +26,9 @@ export class SharpTransform implements AssetTransformationStrategy {
 
     async transform(assetRequest: AssetRequest, asset: Asset): Promise<Asset> {
         if (!utils.SUPPORTED_TRANSFORMABLE_IMAGES.includes(asset.getExtension())) {
+            console.log(
+                `Transformations/optimizations of ${asset.getContentType()} assets are not supported. Skipping.`
+            );
             return asset;
         }
 
@@ -45,6 +48,7 @@ export class SharpTransform implements AssetTransformationStrategy {
     }
 
     private async transformAsset(asset: Asset, options: Omit<AssetRequestOptions, "original">) {
+        console.log("Transform asset", options);
         if (options.width) {
             const { s3, bucket } = this.params;
 
@@ -63,7 +67,15 @@ export class SharpTransform implements AssetTransformationStrategy {
 
                 const buffer = Buffer.from(await Body.transformToByteArray());
 
-                asset.setContentsReader(new CallableContentsReader(() => buffer));
+                const newAsset = asset.withProps({ size: buffer.length });
+                newAsset.setContentsReader(new CallableContentsReader(() => buffer));
+
+                console.log(`Return a previously transformed asset`, {
+                    key: transformedAssetKey,
+                    size: newAsset.getSize()
+                });
+
+                return newAsset;
             } catch (e) {
                 const optimizedImage = await this.optimizeAsset(asset);
 
@@ -73,23 +85,34 @@ export class SharpTransform implements AssetTransformationStrategy {
                 /**
                  * `width` is the only transformation we currently support.
                  */
+                console.log(`Resize the asset (width: ${width})`);
                 const buffer = await optimizedImage.getContents();
-                const transformedBuffer = sharp(buffer, { animated: this.isAssetAnimated(asset) })
+                const transformedBuffer = await sharp(buffer, {
+                    animated: this.isAssetAnimated(asset)
+                })
                     .withMetadata()
                     .resize({ width, withoutEnlargement: true })
                     .toBuffer();
 
                 /**
                  * Transformations are applied to the optimized image.
                  */
-                asset.setContentsReader(new CallableContentsReader(() => transformedBuffer));
+                const newAsset = asset.withProps({ size: transformedBuffer.length });
+                newAsset.setContentsReader(new CallableContentsReader(() => transformedBuffer));
 
                 await s3.putObject({
                     Bucket: bucket,
                     Key: transformedAssetKey,
-                    ContentType: asset.getContentType(),
-                    Body: await asset.getContents()
+                    ContentType: newAsset.getContentType(),
+                    Body: await newAsset.getContents()
                 });
+
+                console.log(`Return the resized asset`, {
+                    key: transformedAssetKey,
+                    size: newAsset.getSize()
+                });
+
+                return newAsset;
             }
         }
 
@@ -99,6 +122,13 @@ export class SharpTransform implements AssetTransformationStrategy {
     private async optimizeAsset(asset: Asset) {
         const { s3, bucket } = this.params;
 
+        console.log("Optimize asset", {
+            id: asset.getId(),
+            key: asset.getKey(),
+            size: asset.getSize(),
+            type: asset.getContentType()
+        });
+
         const assetKey = new AssetKeyGenerator(asset);
         const optimizedAssetKey = assetKey.getOptimizedImageKey();
 
@@ -112,10 +142,16 @@ export class SharpTransform implements AssetTransformationStrategy {
                 throw new Error(`Missing image body!`);
             }
 
+            console.log("Return a previously optimized asset", optimizedAssetKey);
+
             const buffer = Buffer.from(await Body.transformToByteArray());
 
-            asset.setContentsReader(new CallableContentsReader(() => buffer));
+            const newAsset = asset.withProps({ size: buffer.length });
+            newAsset.setContentsReader(new CallableContentsReader(() => buffer));
+
+            return newAsset;
         } catch (e) {
+            console.log("Create an optimized version of the original asset", asset.getKey());
             // If not found, create an optimized version of the original asset.
             const buffer = await asset.getContents();
 
@@ -128,23 +164,26 @@ export class SharpTransform implements AssetTransformationStrategy {
             const optimization = optimizationMap[asset.getContentType()];
 
             if (!optimization) {
-                console.log(`no optimizations defined for ${asset.getContentType()}`);
+                console.log(`No optimizations defined for ${asset.getContentType()}`);
                 return asset;
             }
 
-            const optimizedBuffer = optimization(buffer).toBuffer();
+            const optimizedBuffer = await optimization(buffer).toBuffer();
+
+            console.log("Optimized asset size", optimizedBuffer.length);
 
-            asset.setContentsReader(new CallableContentsReader(() => optimizedBuffer));
+            const newAsset = asset.withProps({ size: optimizedBuffer.length });
+            newAsset.setContentsReader(new CallableContentsReader(() => optimizedBuffer));
 
             await s3.putObject({
                 Bucket: bucket,
                 Key: optimizedAssetKey,
-                ContentType: asset.getContentType(),
-                Body: await asset.getContents()
+                ContentType: newAsset.getContentType(),
+                Body: await newAsset.getContents()
             });
-        }
 
-        return asset;
+            return newAsset;
+        }
     }
 
     private isAssetAnimated(asset: Asset) {

packages/api-file-manager/src/delivery/AssetDelivery/Asset.ts

@@ -21,10 +21,14 @@ export class Asset {
     }
 
     clone() {
-        const clonedAsset = new Asset(structuredClone(this.props));
-        clonedAsset.outputStrategy = this.outputStrategy;
-        clonedAsset.contentsReader = this.contentsReader;
-        return clonedAsset;
+        return this.withProps(structuredClone(this.props));
+    }
+
+    withProps(props: Partial<AssetData>) {
+        const newAsset = new Asset({ ...this.props, ...props });
+        newAsset.contentsReader = this.contentsReader;
+        newAsset.outputStrategy = this.outputStrategy;
+        return newAsset;
     }
 
     getId() {
@@ -39,9 +43,8 @@ export class Asset {
     getKey() {
         return this.props.key;
     }
-    async getSize() {
-        const buffer = await this.getContents();
-        return buffer.length;
+    getSize() {
+        return this.props.size;
     }
     getContentType() {
         return this.props.contentType;

packages/api-file-manager/src/delivery/AssetDelivery/FilesAssetRequestResolver.ts

@@ -1,6 +1,6 @@
 import { Request } from "@webiny/handler/types";
 import { AssetRequestResolver } from "./abstractions/AssetRequestResolver";
-import { AssetRequest } from "./AssetRequest";
+import { AssetRequest, AssetRequestOptions } from "./AssetRequest";
 
 export class FilesAssetRequestResolver implements AssetRequestResolver {
     async resolve(request: Request): Promise<AssetRequest | undefined> {
@@ -15,15 +15,21 @@ export class FilesAssetRequestResolver implements AssetRequestResolver {
         // Example: { '*': '/files/65722cb5c7824a0008d05963/image-48.jpg' },
         const path = params["*"];
 
+        const options: AssetRequestOptions = {
+            ...query,
+            original: "original" in query
+        };
+
+        if (query.width) {
+            options.width = parseInt(query.width);
+        }
+
         return new AssetRequest({
             key: decodeURI(path).replace("/files/", ""),
             context: {
                 url: request.url
             },
-            options: {
-                ...query,
-                width: query.width ? parseInt(query.width) : undefined
-            }
+            options
         });
     }
 }

packages/api-file-manager/src/delivery/AssetDelivery/transformation/TransformationAssetProcessor.ts

@@ -12,6 +12,7 @@ export class TransformationAssetProcessor implements AssetProcessor {
 
         // If the `original` image was requested, we skip all transformations.
         if (original) {
+            console.log("Skip transformations; original asset was requested.");
             return asset;
         }
 

packages/api-file-manager/src/delivery/setupAssetDelivery.ts

@@ -146,6 +146,7 @@ export const setupAssetDelivery = (params: AssetDeliveryParams) => {
                         );
 
                         // Get reply object (runs the output strategy under the hood).
+                        console.log(`Output asset (size: ${processedAsset.getSize()} bytes).`);
                         return outputAsset(reply, processedAsset);
                     },
                     { override: true }