diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab1832b..29bc8f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * New datastore option to ignore Redis cache when downloading media served by a `publicBaseUrl`. This can help ensure more requests get redirected to the CDN. * `HEAD /download` is now supported, as per [MSC4120](https://github.com/matrix-org/matrix-spec-proposals/pull/4120). * S3 datastores can now specify a `prefixLength` to improve S3 performance on some providers. See `config.sample.yaml` for details. +* Add `multipartUploads` flag for running MMR against unsupported S3 providers. See `config.sample.yaml` for details. ### Fixed diff --git a/config.sample.yaml b/config.sample.yaml index b46f2946..f87eef79 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -216,6 +216,12 @@ datastores: # help improve download speeds in some S3 providers. Should not be set to higher than # 16 to avoid future incompatibilities with MMR. Defaults to zero (no prefix). #prefixLength: 3 + # Set to false to disable multipart uploads. This may be required for some *unsupported* + # S3 providers. Note that performance may be significantly degraded with this option set + # to false - installations with multipart uploads disabled will not receive support, + # particularly for performance concerns. If you are using AWS, DigitalOcean Spaces, or + # MinIO, you do not need to set or change this option - your environment is supported. + #multipartUploads: true # Options for controlling archives. Archives are exports of a particular user's content for # the purpose of GDPR or moving media to a different server. diff --git a/datastores/s3.go b/datastores/s3.go index 1e4e6b56..99f15ae3 100644 --- a/datastores/s3.go +++ b/datastores/s3.go @@ -23,6 +23,7 @@ type s3 struct { publicBaseUrl string redirectWhenCached bool prefixLength int + multipartUploads bool } func ResetS3Clients() { @@ -44,6 +45,7 @@ func getS3(ds config.DatastoreConfig) (*s3, error) { publicBaseUrl := ds.Options["publicBaseUrl"] redirectWhenCachedStr, hasRedirectWhenCached := ds.Options["redirectWhenCached"] prefixLengthStr, hasPrefixLength := ds.Options["prefixLength"] + useMultipartStr, hasMultipart := ds.Options["multipartUploads"] if !hasStorageClass { storageClass = "STANDARD" @@ -54,6 +56,11 @@ func getS3(ds config.DatastoreConfig) (*s3, error) { useSsl, _ = strconv.ParseBool(useSslStr) } + useMultipart := true + if hasMultipart && useMultipartStr != "" { + useMultipart, _ = strconv.ParseBool(useMultipartStr) + } + redirectWhenCached := false if hasRedirectWhenCached && redirectWhenCachedStr != "" { redirectWhenCached, _ = strconv.ParseBool(redirectWhenCachedStr) @@ -88,6 +95,7 @@ func getS3(ds config.DatastoreConfig) (*s3, error) { publicBaseUrl: publicBaseUrl, redirectWhenCached: redirectWhenCached, prefixLength: prefixLength, + multipartUploads: useMultipart, } s3clients.Store(ds.Id, s3c) return s3c, nil diff --git a/datastores/upload.go b/datastores/upload.go index 39e403da..d365deea 100644 --- a/datastores/upload.go +++ b/datastores/upload.go @@ -44,7 +44,11 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc() var info minio.UploadInfo - info, err = s3c.client.PutObject(ctx.Context, s3c.bucket, objectName, tee, size, minio.PutObjectOptions{StorageClass: s3c.storageClass, ContentType: contentType}) + info, err = s3c.client.PutObject(ctx.Context, s3c.bucket, objectName, tee, size, minio.PutObjectOptions{ + StorageClass: s3c.storageClass, + ContentType: contentType, + DisableMultipart: !s3c.multipartUploads, + }) uploadedBytes = info.Size } else if ds.Type == "file" { basePath := ds.Options["path"]